Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feature request pipeline to fetch upload data to hugging face (#1239)

koch3092c16af267739c
diff --git a/camel/datahubs/__init__.py b/camel/datahubs/__init__.py
new file mode 100644
index 000000000..5c2cfb3f3
--- /dev/null
+++ b/camel/datahubs/__init__.py
@@ -0,0 +1,23 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from .base import BaseDatasetManager
+from .huggingface import HuggingFaceDatasetManager
+from .models import Record
+
+__all__ = [
+ "BaseDatasetManager",
+ "Record",
+ "HuggingFaceDatasetManager",
+]
diff --git a/camel/datahubs/base.py b/camel/datahubs/base.py
new file mode 100644
index 000000000..6b1e26e8f
--- /dev/null
+++ b/camel/datahubs/base.py
@@ -0,0 +1,136 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Any, List
+
+from camel.datahubs.models import Record
+
+
+class BaseDatasetManager(ABC):
+ r"""Abstract base class for dataset managers."""
+
+ @abstractmethod
+ def create_dataset(self, name: str, **kwargs: Any) -> str:
+ r"""Creates a new dataset.
+
+ Args:
+ name (str): The name of the dataset.
+ kwargs (Any): Additional keyword arguments.
+
+ Returns:
+ str: The URL of the created dataset.
+ """
+ pass
+
+ @abstractmethod
+ def list_datasets(
+ self, username: str, limit: int = 100, **kwargs: Any
+ ) -> List[str]:
+ r"""Lists all datasets for the current user.
+
+ Args:
+ username (str): The username of the user whose datasets to list.
+ limit (int): The maximum number of datasets to list.
+ (default::obj:`100`)
+ kwargs (Any): Additional keyword arguments.
+
+ Returns:
+ List[str]: A list of dataset ids.
+ """
+ pass
+
+ @abstractmethod
+ def delete_dataset(self, dataset_name: str, **kwargs: Any) -> None:
+ r"""Deletes a dataset.
+
+ Args:
+ dataset_name (str): The name of the dataset to delete.
+ kwargs (Any): Additional keyword arguments.
+ """
+ pass
+
+ @abstractmethod
+ def add_records(
+ self,
+ dataset_name: str,
+ records: List[Record],
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> None:
+ r"""Adds records to a dataset.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ records (List[Record]): A list of records to add to the dataset.
+ filepath (str): The path to the file containing the records.
+ (default::obj:`"records/records.json"`)
+ kwargs (Any): Additional keyword arguments.
+ """
+ pass
+
+ @abstractmethod
+ def update_records(
+ self,
+ dataset_name: str,
+ records: List[Record],
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> None:
+ r"""Updates records in a dataset.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ records (List[Record]): A list of records to update in the dataset.
+ filepath (str): The path to the file containing the records.
+ (default::obj:`"records/records.json"`)
+ kwargs (Any): Additional keyword arguments.
+ """
+ pass
+
+ @abstractmethod
+ def list_records(
+ self,
+ dataset_name: str,
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> List[Record]:
+ r"""Lists records in a dataset.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ filepath (str): The path to the file containing the records.
+ (default::obj:`"records/records.json"`)
+ kwargs (Any): Additional keyword arguments.
+ """
+ pass
+
+ # New method for record deletion
+ @abstractmethod
+ def delete_record(
+ self,
+ dataset_name: str,
+ record_id: str,
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> None:
+ r"""Deletes a record from the dataset.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ record_id (str): The ID of the record to delete.
+ filepath (str): The path to the file containing the records.
+ (default::obj:`"records/records.json"`)
+ kwargs (Any): Additional keyword arguments.
+ """
+ pass
diff --git a/camel/datahubs/huggingface.py b/camel/datahubs/huggingface.py
new file mode 100644
index 000000000..bda252876
--- /dev/null
+++ b/camel/datahubs/huggingface.py
@@ -0,0 +1,433 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import os
+import tempfile
+from typing import Any, List, Optional
+
+from camel.datahubs.base import BaseDatasetManager
+from camel.datahubs.models import Record
+from camel.logger import get_logger
+from camel.types import HuggingFaceRepoType
+from camel.utils import api_keys_required, dependencies_required
+
+logger = get_logger(__name__)
+
+
+class HuggingFaceDatasetManager(BaseDatasetManager):
+ r"""A dataset manager for Hugging Face datasets. This class provides
+ methods to create, add, update, delete, and list records in a dataset on
+ the Hugging Face Hub.
+
+ Args:
+ token (str): The Hugging Face API token. If not provided, the token
+ will be read from the environment variable `HUGGING_FACE_TOKEN`.
+ """
+
+ @api_keys_required("HUGGING_FACE_TOKEN")
+ @dependencies_required('huggingface_hub')
+ def __init__(self, token: Optional[str] = None):
+ from huggingface_hub import HfApi
+
+ self._api_key = token or os.getenv("HUGGING_FACE_TOKEN")
+ self.api = HfApi(token=self._api_key)
+
+ def create_dataset_card(
+ self,
+ dataset_name: str,
+ description: str,
+ license: Optional[str] = None,
+ version: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ authors: Optional[List[str]] = None,
+ size_category: Optional[List[str]] = None,
+ language: Optional[List[str]] = None,
+ task_categories: Optional[List[str]] = None,
+ content: Optional[str] = None,
+ ) -> None:
+ r"""Creates and uploads a dataset card to the Hugging Face Hub in YAML
+ format.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ description (str): A description of the dataset.
+ license (str): The license of the dataset. (default: :obj:`None`)
+ version (str): The version of the dataset. (default: :obj:`None`)
+ tags (list): A list of tags for the dataset.(default: :obj:`None`)
+ authors (list): A list of authors of the dataset. (default:
+ :obj:`None`)
+ size_category (list): A size category for the dataset. (default:
+ :obj:`None`)
+ language (list): A list of languages the dataset is in. (default:
+ :obj:`None`)
+ task_categories (list): A list of task categories. (default:
+ :obj:`None`)
+ content (str): Custom markdown content that the user wants to add
+ to the dataset card. (default: :obj:`None`)
+ """
+ import yaml
+
+ metadata = {
+ "license": license,
+ "authors": authors,
+ "task_categories": task_categories,
+ "language": language,
+ "tags": tags,
+ "pretty_name": dataset_name,
+ "size_categories": size_category,
+ "version": version,
+ "description": description,
+ }
+
+ # Remove keys with None values
+ metadata = {k: v for k, v in metadata.items() if v}
+
+ card_content = (
+ "---\n"
+ + yaml.dump(metadata, default_flow_style=False, allow_unicode=True)
+ + "\n---"
+ )
+
+ if content:
+ card_content += f"\n\n# Additional Information\n{content}\n"
+
+ self._upload_file(
+ file_content=card_content,
+ dataset_name=dataset_name,
+ filepath="README.md",
+ file_type="md",
+ )
+
+ def create_dataset(
+ self, name: str, private: bool = False, **kwargs: Any
+ ) -> str:
+ r"""Creates a new dataset on the Hugging Face Hub.
+
+ Args:
+ name (str): The name of the dataset.
+ private (bool): Whether the dataset should be private. defaults to
+ False.
+ kwargs (Any): Additional keyword arguments.
+
+ Returns:
+ str: The URL of the created dataset.
+ """
+ from huggingface_hub.errors import RepositoryNotFoundError
+
+ try:
+ self.api.repo_info(
+ repo_id=name,
+ repo_type=HuggingFaceRepoType.DATASET.value,
+ **kwargs,
+ )
+ except RepositoryNotFoundError:
+ self.api.create_repo(
+ repo_id=name,
+ repo_type=HuggingFaceRepoType.DATASET.value,
+ private=private,
+ )
+
+ return f"https://huggingface.co/datasets/{name}"
+
+ def list_datasets(
+ self, username: str, limit: int = 100, **kwargs: Any
+ ) -> List[str]:
+ r"""Lists all datasets for the current user.
+
+ Args:
+ username (str): The username of the user whose datasets to list.
+ limit (int): The maximum number of datasets to list.
+ (default: :obj:`100`)
+ kwargs (Any): Additional keyword arguments.
+
+ Returns:
+ List[str]: A list of dataset ids.
+ """
+ try:
+ return [
+ dataset.id
+ for dataset in self.api.list_datasets(
+ author=username, limit=limit, **kwargs
+ )
+ ]
+ except Exception as e:
+ logger.error(f"Error listing datasets: {e}")
+ return []
+
+ def delete_dataset(self, dataset_name: str, **kwargs: Any) -> None:
+ r"""Deletes a dataset from the Hugging Face Hub.
+
+ Args:
+ dataset_name (str): The name of the dataset to delete.
+ kwargs (Any): Additional keyword arguments.
+ """
+ try:
+ self.api.delete_repo(
+ repo_id=dataset_name,
+ repo_type=HuggingFaceRepoType.DATASET.value,
+ **kwargs,
+ )
+ logger.info(f"Dataset '{dataset_name}' deleted successfully.")
+ except Exception as e:
+ logger.error(f"Error deleting dataset '{dataset_name}': {e}")
+ raise
+
+ def add_records(
+ self,
+ dataset_name: str,
+ records: List[Record],
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> None:
+ r"""Adds records to a dataset on the Hugging Face Hub.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ records (List[Record]): A list of records to add to the dataset.
+ filepath (str): The path to the file containing the records.
+ kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ ValueError: If the dataset already has a records file.
+ """
+ existing_records = self._download_records(
+ dataset_name=dataset_name, filepath=filepath, **kwargs
+ )
+
+ if existing_records:
+ raise ValueError(
+ f"Dataset '{filepath}' already exists. "
+ f"Use `update_records` to modify."
+ )
+
+ self._upload_records(
+ records=records,
+ dataset_name=dataset_name,
+ filepath=filepath,
+ **kwargs,
+ )
+
+ def update_records(
+ self,
+ dataset_name: str,
+ records: List[Record],
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> None:
+ r"""Updates records in a dataset on the Hugging Face Hub.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ records (List[Record]): A list of records to update in the dataset.
+ filepath (str): The path to the file containing the records.
+ kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ ValueError: If the dataset does not have an existing file to update
+ records in.
+ """
+ existing_records = self._download_records(
+ dataset_name=dataset_name, filepath=filepath, **kwargs
+ )
+
+ if not existing_records:
+ logger.warning(
+ f"Dataset '{dataset_name}' does not have existing "
+ "records. Adding new records."
+ )
+ self._upload_records(
+ records=records,
+ dataset_name=dataset_name,
+ filepath=filepath,
+ **kwargs,
+ )
+ return
+
+ old_dict = {record.id: record for record in existing_records}
+ new_dict = {record.id: record for record in records}
+ merged_dict = old_dict.copy()
+ merged_dict.update(new_dict)
+
+ self._upload_records(
+ records=list(merged_dict.values()),
+ dataset_name=dataset_name,
+ filepath=filepath,
+ **kwargs,
+ )
+
+ def delete_record(
+ self,
+ dataset_name: str,
+ record_id: str,
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> None:
+ r"""Deletes a record from the dataset.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ record_id (str): The ID of the record to delete.
+ filepath (str): The path to the file containing the records.
+ kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ ValueError: If the dataset does not have an existing file to delete
+ records from.
+ """
+ existing_records = self._download_records(
+ dataset_name=dataset_name, filepath=filepath, **kwargs
+ )
+
+ if not existing_records:
+ raise ValueError(
+ f"Dataset '{dataset_name}' does not have an existing file to "
+ f"delete records from."
+ )
+
+ filtered_records = [
+ record for record in existing_records if record.id != record_id
+ ]
+
+ self._upload_records(
+ records=filtered_records,
+ dataset_name=dataset_name,
+ filepath=filepath,
+ **kwargs,
+ )
+
+ def list_records(
+ self,
+ dataset_name: str,
+ filepath: str = "records/records.json",
+ **kwargs: Any,
+ ) -> List[Record]:
+ r"""Lists all records in a dataset.
+
+ Args:
+ dataset_name (str): The name of the dataset.
+ filepath (str): The path to the file containing the records.
+ kwargs (Any): Additional keyword arguments.
+
+ Returns:
+ List[Record]: A list of records in the dataset.
+ """
+ return self._download_records(
+ dataset_name=dataset_name, filepath=filepath, **kwargs
+ )
+
+ def _download_records(
+ self, dataset_name: str, filepath: str, **kwargs: Any
+ ) -> List[Record]:
+ from huggingface_hub import hf_hub_download
+ from huggingface_hub.errors import EntryNotFoundError
+
+ try:
+ downloaded_file_path = hf_hub_download(
+ repo_id=dataset_name,
+ filename=filepath,
+ repo_type=HuggingFaceRepoType.DATASET.value,
+ token=self._api_key,
+ **kwargs,
+ )
+
+ with open(downloaded_file_path, "r") as f:
+ records_data = json.load(f)
+
+ return [Record(**record) for record in records_data]
+ except EntryNotFoundError:
+ logger.info(f"No records found for dataset '{dataset_name}'.")
+ return []
+ except Exception as e:
+ logger.error(f"Error downloading or processing records: {e}")
+ raise e
+
+ def _upload_records(
+ self,
+ records: List[Record],
+ dataset_name: str,
+ filepath: str,
+ **kwargs: Any,
+ ):
+ with tempfile.NamedTemporaryFile(
+ delete=False, mode="w", newline="", encoding="utf-8"
+ ) as f:
+ json.dump([record.model_dump() for record in records], f)
+ temp_file_path = f.name
+
+ try:
+ self.api.upload_file(
+ path_or_fileobj=temp_file_path,
+ path_in_repo=filepath,
+ repo_id=dataset_name,
+ repo_type=HuggingFaceRepoType.DATASET.value,
+ **kwargs,
+ )
+ except Exception as e:
+ logger.error(f"Error uploading records file: {e}")
+ raise
+ finally:
+ if os.path.exists(temp_file_path):
+ os.remove(temp_file_path)
+
+ def _upload_file(
+ self,
+ file_content: str,
+ dataset_name: str,
+ filepath: str,
+ file_type: str = "json",
+ **kwargs: Any,
+ ):
+ with tempfile.NamedTemporaryFile(
+ mode="w", delete=False, suffix=f".{file_type}"
+ ) as f:
+ if file_type == "json":
+ if isinstance(file_content, str):
+ try:
+ json_content = json.loads(file_content)
+ except json.JSONDecodeError:
+ raise ValueError(
+ "Invalid JSON string provided for file_content."
+ )
+ else:
+ try:
+ json.dumps(file_content)
+ json_content = file_content
+ except (TypeError, ValueError):
+ raise ValueError(
+ "file_content is not JSON serializable."
+ )
+
+ json.dump(json_content, f)
+ elif file_type == "md" or file_type == "txt":
+ f.write(file_content)
+ else:
+ raise ValueError(f"Unsupported file type: {file_type}")
+
+ temp_file_path = f.name
+
+ try:
+ self.api.upload_file(
+ path_or_fileobj=temp_file_path,
+ path_in_repo=filepath,
+ repo_id=dataset_name,
+ repo_type=HuggingFaceRepoType.DATASET.value,
+ **kwargs,
+ )
+ logger.info(f"File uploaded successfully: {filepath}")
+ except Exception as e:
+ logger.error(f"Error uploading file: {e}")
+ raise
+
+ if os.path.exists(temp_file_path):
+ os.remove(temp_file_path)
diff --git a/camel/datahubs/models.py b/camel/datahubs/models.py
new file mode 100644
index 000000000..cd63de8f2
--- /dev/null
+++ b/camel/datahubs/models.py
@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel
+
+
+class Record(BaseModel):
+ id: Optional[str] = None
+ metadata: Optional[Dict[str, Any]] = None
+ content: Dict[str, Any]
diff --git a/camel/types/__init__.py b/camel/types/__init__.py
index 7c6a3dcab..3904dc261 100644
--- a/camel/types/__init__.py
+++ b/camel/types/__init__.py
@@ -14,6 +14,7 @@
from .enums import (
AudioModelType,
EmbeddingModelType,
+ HuggingFaceRepoType,
ModelPlatformType,
ModelType,
OpenAIBackendRole,
@@ -73,4 +74,5 @@ __all__ = [
'NOT_GIVEN',
'NotGiven',
'ParsedChatCompletion',
+ 'HuggingFaceRepoType',
]
diff --git a/camel/types/enums.py b/camel/types/enums.py
index c623a683b..5cca71285 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -749,3 +749,9 @@ class JinaReturnFormat(Enum):
MARKDOWN = "markdown"
HTML = "html"
TEXT = "text"
+
+
+class HuggingFaceRepoType(str, Enum):
+ DATASET = "dataset"
+ MODEL = "model"
+ SPACE = "space"
diff --git a/examples/datahubs/huggingface.py b/examples/datahubs/huggingface.py
new file mode 100644
index 000000000..8f26e9137
--- /dev/null
+++ b/examples/datahubs/huggingface.py
@@ -0,0 +1,117 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from camel.datahubs.huggingface import HuggingFaceDatasetManager
+from camel.datahubs.models import Record
+
+manager = HuggingFaceDatasetManager()
+
+USERNAME = "username"
+REPO_ID = f"{USERNAME}/test-dataset-example"
+
+records = [
+ Record(
+ id="record-1",
+ content={"input": "What is AI?", "output": "Artificial Intelligence"},
+ metadata={"method": "SFT"},
+ ),
+ Record(
+ id="record-2",
+ content={"input": "Translate 'hello'", "output": "Bonjour"},
+ metadata={"method": "GPT"},
+ ),
+]
+
+# 1. create a dataset
+print("Creating dataset...")
+dataset_url = manager.create_dataset(name=REPO_ID)
+print(f"Dataset created: {dataset_url}")
+
+# 2. create a dataset card
+print("Creating dataset card...")
+manager.create_dataset_card(
+ dataset_name=REPO_ID,
+ description="Test dataset",
+ license="mit",
+ language=["en"],
+ size_category="<1MB",
+ version="0.1.0",
+ tags=["test", "example"],
+ task_categories=["other"],
+ authors=["camel-ai"],
+)
+print("Dataset card created successfully.")
+
+# 3. add records to the dataset
+print("Adding records to the dataset...")
+manager.add_records(dataset_name=REPO_ID, records=records)
+print("Records added successfully.")
+
+# 4. list all records
+print("Listing all records...")
+all_records = manager.list_records(dataset_name=REPO_ID)
+print("Records in the dataset:")
+for record in all_records:
+ print(
+ f"- ID: {record.id}, Input: {record.content['input']}, "
+ f"Output: {record.content['output']}"
+ )
+
+# 5. update a record
+new_records = [
+ Record(
+ id="record-3",
+ content={"input": "What is ML?", "output": "Machine Learning"},
+ metadata={"method": "Updated GPT"},
+ )
+]
+print("Updating records...")
+manager.update_records(dataset_name=REPO_ID, records=new_records)
+print("Records updated successfully.")
+
+# 6. list updated records
+print("Listing updated records...")
+updated_records = manager.list_records(dataset_name=REPO_ID)
+print("Updated records in the dataset:")
+for record in updated_records:
+ print(
+ f"- ID: {record.id}, Input: {record.content['input']}, "
+ f"Output: {record.content['output']}"
+ )
+
+# 7. delete a record
+print("Deleting record with ID 'record-1'...")
+manager.delete_record(dataset_name=REPO_ID, record_id="record-1")
+print("Record deleted successfully.")
+
+# 8. list records after deletion
+print("Listing records after deletion...")
+final_records = manager.list_records(dataset_name=REPO_ID)
+print("Final records in the dataset:")
+for record in final_records:
+ print(
+ f"- ID: {record.id}, Input: {record.content['input']}, "
+ f"Output: {record.content['output']}"
+ )
+
+# 9. list all datasets
+print("Listing all datasets...")
+datasets = manager.list_datasets(USERNAME)
+print("Datasets:")
+for dataset in datasets:
+ print(f"- {dataset}")
+
+# 10. delete a dataset
+print(f"Deleting dataset: {REPO_ID}...")
+manager.delete_dataset(dataset_name=REPO_ID)
+print("Dataset deleted successfully.")
diff --git a/poetry.lock b/poetry.lock
index 4da1bab0b..d66fd9acd 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
[[package]]
name = "accelerate"
@@ -2149,12 +2149,12 @@ files = [
google-auth = ">=2.14.1,<3.0.dev0"
googleapis-common-protos = ">=1.56.2,<2.0.dev0"
grpcio = [
- {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},
{version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
+ {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},
]
grpcio-status = [
- {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},
{version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
+ {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},
]
proto-plus = ">=1.22.3,<2.0.0dev"
protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0"
@@ -3916,6 +3916,7 @@ python-versions = ">=3.7"
files = [
{file = "milvus_lite-2.4.10-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:fc4246d3ed7d1910847afce0c9ba18212e93a6e9b8406048436940578dfad5cb"},
{file = "milvus_lite-2.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:74a8e07c5e3b057df17fbb46913388e84df1dc403a200f4e423799a58184c800"},
+ {file = "milvus_lite-2.4.10-py3-none-manylinux2014_aarch64.whl", hash = "sha256:240c7386b747bad696ecb5bd1f58d491e86b9d4b92dccee3315ed7256256eddc"},
{file = "milvus_lite-2.4.10-py3-none-manylinux2014_x86_64.whl", hash = "sha256:211d2e334a043f9282bdd9755f76b9b2d93b23bffa7af240919ffce6a8dfe325"},
]
@@ -4882,9 +4883,9 @@ files = [
[package.dependencies]
numpy = [
+ {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""},
{version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""},
{version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""},
- {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""},
{version = ">=1.26.0", markers = "python_version >= \"3.12\""},
]
@@ -5064,8 +5065,8 @@ files = [
[package.dependencies]
numpy = [
- {version = ">=1.22.4", markers = "python_version < \"3.11\""},
{version = ">=1.23.2", markers = "python_version == \"3.11\""},
+ {version = ">=1.22.4", markers = "python_version < \"3.11\""},
{version = ">=1.26.0", markers = "python_version >= \"3.12\""},
]
python-dateutil = ">=2.8.2"
@@ -5910,6 +5911,7 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs
optional = true
python-versions = ">=3.8"
files = [
+ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},
{file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},
]
@@ -5920,6 +5922,7 @@ description = "A collection of ASN.1-based protocols modules"
optional = true
python-versions = ">=3.8"
files = [
+ {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},
{file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},
]
@@ -7357,40 +7360,30 @@ files = [
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a606ef75a60ecf3d924613892cc603b154178ee25abb3055db5062da811fd969"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd5415dded15c3822597455bc02bcd66e81ef8b7a48cb71a33628fc9fdde39df"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"},
- {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"},
- {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d84318609196d6bd6da0edfa25cedfbabd8dbde5140a0a23af29ad4b8f91fb1e"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb43a269eb827806502c7c8efb7ae7e9e9d0573257a46e8e952f4d4caba4f31e"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"},
- {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"},
- {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"},
- {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"},
- {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:e7e3736715fbf53e9be2a79eb4db68e4ed857017344d697e8b9749444ae57475"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7e75b4965e1d4690e93021adfcecccbca7d61c7bddd8e22406ef2ff20d74ef"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"},
- {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"},
- {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bc5f1e1c28e966d61d2519f2a3d451ba989f9ea0f2307de7bc45baa526de9e45"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a0e060aace4c24dcaf71023bbd7d42674e3b230f7e7b97317baf1e953e5b519"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"},
- {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"},
- {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"},
{file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"},
@@ -8893,6 +8886,17 @@ files = [
{file = "types_python_dateutil-2.9.0.20241206.tar.gz", hash = "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb"},
]
+[[package]]
+name = "types-pyyaml"
+version = "6.0.12.20240917"
+description = "Typing stubs for PyYAML"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"},
+ {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"},
+]
+
[[package]]
name = "types-regex"
version = "2024.11.6.20241108"
@@ -9885,7 +9889,7 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",
type = ["pytest-mypy"]
[extras]
-all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]
+all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "pyyaml", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]
encoders = ["sentence-transformers"]
graph-storages = ["nebula3-python", "neo4j"]
huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "torch", "transformers"]
@@ -9897,10 +9901,10 @@ retrievers = ["cohere", "rank-bm25"]
runtime = ["docker"]
search-tools = ["duckduckgo-search", "tavily-python", "wikipedia", "wolframalpha"]
test = ["mock", "pytest", "pytest-asyncio"]
-tools = ["PyMuPDF", "agentops", "apify_client", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "datacommons", "datacommons_pandas", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]
+tools = ["PyMuPDF", "agentops", "apify_client", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "datacommons", "datacommons_pandas", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "pyyaml", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]
vector-databases = ["pymilvus", "qdrant-client"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.13"
-content-hash = "0f3fa1b2e436a1dafb85b426e0d863b6563e4aa9880f92be2af84e7946bd231d"
+content-hash = "66d081b88fb592beaee2ce10058723920f82106a987f90490f6d75b746e43085"
diff --git a/pyproject.toml b/pyproject.toml
index d5bdd2763..04c740079 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -109,6 +109,7 @@ apify_client = { version = "^1.8.1", optional = true }
notion-client = { version = "^2.2.1", optional = true }
yt-dlp = { version = "^2024.11.4", optional = true }
ffmpeg-python = { version = "^0.2.0", optional = true }
+pyyaml = { version = "^6.0.2", optional = true }
# encoders
@@ -220,6 +221,7 @@ tools = [
"notion-client",
"yt-dlp",
"ffmpeg-python",
+ "pyyaml",
]
vector-databases = [
@@ -301,6 +303,7 @@ all = [
"notion-client",
"yt-dlp",
"ffmpeg-python",
+ "pyyaml",
# vector-database
"qdrant-client",
"pymilvus",
@@ -356,6 +359,7 @@ types-setuptools = "*"
types-tqdm = "*"
types-colorama = "^0"
types-requests = "^2"
+types-PyYAML = "^6"
[tool.poetry.group.docs]
optional = true
@@ -430,6 +434,7 @@ module = [
"database_connection",
"huggingface_hub",
"huggingface_hub.utils._errors",
+ "huggingface_hub.errors",
"wikipedia",
"duckduckgo_search",
"newspaper",
diff --git a/test/datahubs/test_huggingface.py b/test/datahubs/test_huggingface.py
new file mode 100644
index 000000000..8e3a83f28
--- /dev/null
+++ b/test/datahubs/test_huggingface.py
@@ -0,0 +1,296 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import datetime
+import json
+import os
+from unittest.mock import MagicMock, patch
+
+import pytest
+from huggingface_hub.errors import EntryNotFoundError
+
+from camel.datahubs.huggingface import HuggingFaceDatasetManager
+from camel.datahubs.models import Record
+
+TOKEN = "your_huggingface_token"
+DATASET_NAME = (
+ f"test-dataset-{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}"
+)
+REPO_ID = f"username/{DATASET_NAME}"
+
+os.environ["HUGGING_FACE_TOKEN"] = TOKEN
+
+
+def manager():
+ return HuggingFaceDatasetManager()
+
+
+def test_create_dataset(manager):
+ with patch("huggingface_hub.HfApi.create_repo") as mock_create_repo:
+ mock_create_repo.return_value = {"repo_id": REPO_ID}
+
+ dataset_url = manager.create_dataset(name=REPO_ID)
+ assert dataset_url == f"https://huggingface.co/datasets/{REPO_ID}"
+
+ mock_create_repo.assert_called_once_with(
+ repo_id=REPO_ID, repo_type="dataset", private=False
+ )
+
+
+def test_create_dataset_card():
+ manager = HuggingFaceDatasetManager(token="fake-token")
+ dataset_name = "user/test-dataset"
+ description = "Test dataset"
+ license = "MIT"
+ version = "0.1.0"
+ tags = ["test", "example"]
+ authors = ["camel-ai"]
+ size_category = "<1MB"
+ language = ["en"]
+ task_categories = ["other"]
+ content = "Additional information about the dataset."
+
+ with patch(
+ "camel.datahubs.HuggingFaceDatasetManager._upload_file"
+ ) as mock_upload_file:
+ mock_upload_file.return_value = None
+
+ manager.create_dataset_card(
+ dataset_name=dataset_name,
+ description=description,
+ license=license,
+ version=version,
+ tags=tags,
+ authors=authors,
+ size_category=size_category,
+ language=language,
+ task_categories=task_categories,
+ content=content,
+ )
+
+ mock_upload_file.assert_called_once_with(
+ file_content="---\nauthors:\n- camel-ai\ndescription: Test dataset"
+ "\nlanguage:\n- en\nlicense: MIT\npretty_name: "
+ "user/test-dataset\nsize_categories: <1MB\ntags:\n"
+ "- test\n- example\ntask_categories:\n- other\n"
+ "version: 0.1.0\n\n---\n\n# Additional Information\n"
+ "Additional information about the dataset.\n",
+ dataset_name=dataset_name,
+ filepath="README.md",
+ file_type="md",
+ )
+
+
+def test_add_records(manager):
+ records = [
+ Record(
+ id="record-1",
+ content={
+ "input": "What is AI?",
+ "output": "Artificial Intelligence",
+ },
+ metadata={"method": "SFT"},
+ ),
+ Record(
+ id="record-2",
+ content={"input": "Translate 'hello'", "output": "Bonjour"},
+ metadata={"method": "GPT"},
+ ),
+ ]
+
+ with (
+ patch(
+ "huggingface_hub.hf_hub_download",
+ side_effect=EntryNotFoundError("404 Client Error."),
+ ) as mock_download,
+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,
+ ):
+ manager.add_records(dataset_name=REPO_ID, records=records)
+
+ mock_download.assert_called_once()
+ mock_upload_file.assert_called_once()
+ call_args = mock_upload_file.call_args[1]
+ assert call_args["repo_id"] == REPO_ID
+ assert call_args["path_in_repo"] == "records/records.json"
+ assert call_args["repo_type"] == "dataset"
+
+
+def test_update_records(manager):
+ existing_records = [
+ {
+ "id": "record-1",
+ "content": {
+ "input": "What is AI?",
+ "output": "Artificial Intelligence",
+ },
+ "metadata": {"method": "SFT"},
+ }
+ ]
+
+ new_records = [
+ Record(
+ id="record-2",
+ content={"input": "Translate 'hello'", "output": "Salut"},
+ metadata={"method": "Updated GPT"},
+ ),
+ Record(
+ id="record-3",
+ content={"input": "What is ML?", "output": "Machine Learning"},
+ metadata={"method": "SFT"},
+ ),
+ ]
+
+ with (
+ patch("huggingface_hub.hf_hub_download") as mock_download,
+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,
+ patch("builtins.open", new_callable=MagicMock) as mock_open,
+ ):
+ mock_download.return_value = "/mock/path/records.json"
+
+ mock_open.return_value.__enter__.return_value.read.return_value = (
+ json.dumps(existing_records)
+ )
+
+ manager.update_records(dataset_name=REPO_ID, records=new_records)
+
+ mock_upload_file.assert_called_once()
+ call_args = mock_upload_file.call_args[1]
+ assert call_args["repo_id"] == REPO_ID
+ assert call_args["path_in_repo"] == "records/records.json"
+ assert call_args["repo_type"] == "dataset"
+
+
+def test_list_records(manager):
+ mock_records = [
+ {
+ "id": "record-1",
+ "content": {
+ "input": "What is AI?",
+ "output": "Artificial Intelligence",
+ },
+ "metadata": {"method": "SFT"},
+ },
+ {
+ "id": "record-2",
+ "content": {"input": "Translate 'hello'", "output": "Salut"},
+ "metadata": {"method": "GPT"},
+ },
+ ]
+
+ with (
+ patch("huggingface_hub.hf_hub_download") as mock_download,
+ patch("builtins.open", new_callable=MagicMock) as mock_open,
+ ):
+ mock_download.return_value = "/mock/path/records.json"
+ mock_open.return_value.__enter__.return_value.read.return_value = (
+ json.dumps(mock_records)
+ )
+
+ records = manager.list_records(dataset_name=REPO_ID)
+ assert len(records) == len(mock_records)
+
+ for original, loaded in zip(mock_records, records):
+ assert original["id"] == loaded.id
+ assert original["content"] == loaded.content
+ assert original["metadata"] == loaded.metadata
+
+
+def test_add_records_error_on_existing_file(manager):
+ records = [
+ Record(
+ id="record-1",
+ content={
+ "input": "What is AI?",
+ "output": "Artificial Intelligence",
+ },
+ metadata={"method": "SFT"},
+ )
+ ]
+
+ with (
+ patch(
+ "huggingface_hub.hf_hub_download",
+ side_effect=ValueError(
+ "Dataset 'records/records.json' already exists. "
+ "Use `update_records` to modify."
+ ),
+ ) as mock_download,
+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,
+ ):
+ mock_download.return_value = "/mock/path/records.json"
+
+ with pytest.raises(
+ ValueError,
+ match="Dataset 'records/records.json' already exists. "
+ "Use `update_records` to modify.",
+ ):
+ manager.add_records(dataset_name=REPO_ID, records=records)
+
+ mock_upload_file.assert_not_called()
+
+
+def test_list_datasets(manager):
+ with patch("huggingface_hub.HfApi.list_datasets") as mock_list_datasets:
+ mock_dataset = MagicMock()
+ mock_dataset.id = REPO_ID
+ mock_list_datasets.return_value = [mock_dataset]
+
+ datasets = manager.list_datasets(username="username")
+ assert datasets == [REPO_ID]
+
+ mock_list_datasets.assert_called_once()
+
+
+def test_delete_dataset(manager):
+ with patch("huggingface_hub.HfApi.delete_repo") as mock_delete_repo:
+ manager.delete_dataset(dataset_name=REPO_ID)
+ mock_delete_repo.assert_called_once_with(
+ repo_id=REPO_ID, repo_type="dataset"
+ )
+
+
+def test_delete_record(manager):
+ with (
+ patch("huggingface_hub.hf_hub_download") as mock_download,
+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,
+ patch("builtins.open", new_callable=MagicMock) as mock_open,
+ ):
+ mock_download.return_value = "/mock/path/records.json"
+
+ existing_records = [
+ Record(
+ id="record-1",
+ content={
+ "input": "What is AI?",
+ "output": "Artificial Intelligence",
+ },
+ metadata={"method": "SFT"},
+ ),
+ Record(
+ id="record-2",
+ content={"input": "Translate 'hello'", "output": "Bonjour"},
+ metadata={"method": "GPT"},
+ ),
+ ]
+ mock_open.return_value.__enter__.return_value.read.return_value = (
+ json.dumps([record.model_dump() for record in existing_records])
+ )
+
+ manager.delete_record(dataset_name=REPO_ID, record_id="record-1")
+
+ mock_upload_file.assert_called_once()
+ call_args = mock_upload_file.call_args[1]
+ assert call_args["repo_id"] == REPO_ID
+ assert call_args["path_in_repo"] == "records/records.json"
+ assert call_args["repo_type"] == "dataset"