🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feature request pipeline to fetch upload data to hugging face (#1239)
koch3092c16af267739c
diff --git a/camel/datahubs/__init__.py b/camel/datahubs/__init__.pynew file mode 100644index 000000000..5c2cfb3f3--- /dev/null+++ b/camel/datahubs/__init__.py@@ -0,0 +1,23 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from .base import BaseDatasetManager+from .huggingface import HuggingFaceDatasetManager+from .models import Record++__all__ = [+ "BaseDatasetManager",+ "Record",+ "HuggingFaceDatasetManager",+]diff --git a/camel/datahubs/base.py b/camel/datahubs/base.pynew file mode 100644index 000000000..6b1e26e8f--- /dev/null+++ b/camel/datahubs/base.py@@ -0,0 +1,136 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from abc import ABC, abstractmethod+from typing import Any, List++from camel.datahubs.models import Record+++class BaseDatasetManager(ABC):+ r"""Abstract base class for dataset managers."""++ @abstractmethod+ def create_dataset(self, name: str, **kwargs: Any) -> str:+ r"""Creates a new dataset.++ Args:+ name (str): The name of the dataset.+ kwargs (Any): Additional keyword arguments.++ Returns:+ str: The URL of the created dataset.+ """+ pass++ @abstractmethod+ def list_datasets(+ self, username: str, limit: int = 100, **kwargs: Any+ ) -> List[str]:+ r"""Lists all datasets for the current user.++ Args:+ username (str): The username of the user whose datasets to list.+ limit (int): The maximum number of datasets to list.+ (default::obj:`100`)+ kwargs (Any): Additional keyword arguments.++ Returns:+ List[str]: A list of dataset ids.+ """+ pass++ @abstractmethod+ def delete_dataset(self, dataset_name: str, **kwargs: Any) -> None:+ r"""Deletes a dataset.++ Args:+ dataset_name (str): The name of the dataset to delete.+ kwargs (Any): Additional keyword arguments.+ """+ pass++ @abstractmethod+ def add_records(+ self,+ dataset_name: str,+ records: List[Record],+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> None:+ r"""Adds records to a dataset.++ Args:+ dataset_name (str): The name of the dataset.+ records (List[Record]): A list of records to add to the dataset.+ filepath (str): The path to the file containing the records.+ (default::obj:`"records/records.json"`)+ kwargs (Any): Additional keyword arguments.+ """+ pass++ @abstractmethod+ def update_records(+ self,+ dataset_name: str,+ records: List[Record],+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> None:+ r"""Updates records in a dataset.++ Args:+ dataset_name (str): The name of the dataset.+ records (List[Record]): A list of records to update in the dataset.+ filepath (str): The path to the file containing the records.+ (default::obj:`"records/records.json"`)+ kwargs (Any): Additional keyword arguments.+ """+ pass++ @abstractmethod+ def list_records(+ self,+ dataset_name: str,+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> List[Record]:+ r"""Lists records in a dataset.++ Args:+ dataset_name (str): The name of the dataset.+ filepath (str): The path to the file containing the records.+ (default::obj:`"records/records.json"`)+ kwargs (Any): Additional keyword arguments.+ """+ pass++ # New method for record deletion+ @abstractmethod+ def delete_record(+ self,+ dataset_name: str,+ record_id: str,+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> None:+ r"""Deletes a record from the dataset.++ Args:+ dataset_name (str): The name of the dataset.+ record_id (str): The ID of the record to delete.+ filepath (str): The path to the file containing the records.+ (default::obj:`"records/records.json"`)+ kwargs (Any): Additional keyword arguments.+ """+ passdiff --git a/camel/datahubs/huggingface.py b/camel/datahubs/huggingface.pynew file mode 100644index 000000000..bda252876--- /dev/null+++ b/camel/datahubs/huggingface.py@@ -0,0 +1,433 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import json+import os+import tempfile+from typing import Any, List, Optional++from camel.datahubs.base import BaseDatasetManager+from camel.datahubs.models import Record+from camel.logger import get_logger+from camel.types import HuggingFaceRepoType+from camel.utils import api_keys_required, dependencies_required++logger = get_logger(__name__)+++class HuggingFaceDatasetManager(BaseDatasetManager):+ r"""A dataset manager for Hugging Face datasets. This class provides+ methods to create, add, update, delete, and list records in a dataset on+ the Hugging Face Hub.++ Args:+ token (str): The Hugging Face API token. If not provided, the token+ will be read from the environment variable `HUGGING_FACE_TOKEN`.+ """++ @api_keys_required("HUGGING_FACE_TOKEN")+ @dependencies_required('huggingface_hub')+ def __init__(self, token: Optional[str] = None):+ from huggingface_hub import HfApi++ self._api_key = token or os.getenv("HUGGING_FACE_TOKEN")+ self.api = HfApi(token=self._api_key)++ def create_dataset_card(+ self,+ dataset_name: str,+ description: str,+ license: Optional[str] = None,+ version: Optional[str] = None,+ tags: Optional[List[str]] = None,+ authors: Optional[List[str]] = None,+ size_category: Optional[List[str]] = None,+ language: Optional[List[str]] = None,+ task_categories: Optional[List[str]] = None,+ content: Optional[str] = None,+ ) -> None:+ r"""Creates and uploads a dataset card to the Hugging Face Hub in YAML+ format.++ Args:+ dataset_name (str): The name of the dataset.+ description (str): A description of the dataset.+ license (str): The license of the dataset. (default: :obj:`None`)+ version (str): The version of the dataset. (default: :obj:`None`)+ tags (list): A list of tags for the dataset.(default: :obj:`None`)+ authors (list): A list of authors of the dataset. (default:+ :obj:`None`)+ size_category (list): A size category for the dataset. (default:+ :obj:`None`)+ language (list): A list of languages the dataset is in. (default:+ :obj:`None`)+ task_categories (list): A list of task categories. (default:+ :obj:`None`)+ content (str): Custom markdown content that the user wants to add+ to the dataset card. (default: :obj:`None`)+ """+ import yaml++ metadata = {+ "license": license,+ "authors": authors,+ "task_categories": task_categories,+ "language": language,+ "tags": tags,+ "pretty_name": dataset_name,+ "size_categories": size_category,+ "version": version,+ "description": description,+ }++ # Remove keys with None values+ metadata = {k: v for k, v in metadata.items() if v}++ card_content = (+ "---\n"+ + yaml.dump(metadata, default_flow_style=False, allow_unicode=True)+ + "\n---"+ )++ if content:+ card_content += f"\n\n# Additional Information\n{content}\n"++ self._upload_file(+ file_content=card_content,+ dataset_name=dataset_name,+ filepath="README.md",+ file_type="md",+ )++ def create_dataset(+ self, name: str, private: bool = False, **kwargs: Any+ ) -> str:+ r"""Creates a new dataset on the Hugging Face Hub.++ Args:+ name (str): The name of the dataset.+ private (bool): Whether the dataset should be private. defaults to+ False.+ kwargs (Any): Additional keyword arguments.++ Returns:+ str: The URL of the created dataset.+ """+ from huggingface_hub.errors import RepositoryNotFoundError++ try:+ self.api.repo_info(+ repo_id=name,+ repo_type=HuggingFaceRepoType.DATASET.value,+ **kwargs,+ )+ except RepositoryNotFoundError:+ self.api.create_repo(+ repo_id=name,+ repo_type=HuggingFaceRepoType.DATASET.value,+ private=private,+ )++ return f"https://huggingface.co/datasets/{name}"++ def list_datasets(+ self, username: str, limit: int = 100, **kwargs: Any+ ) -> List[str]:+ r"""Lists all datasets for the current user.++ Args:+ username (str): The username of the user whose datasets to list.+ limit (int): The maximum number of datasets to list.+ (default: :obj:`100`)+ kwargs (Any): Additional keyword arguments.++ Returns:+ List[str]: A list of dataset ids.+ """+ try:+ return [+ dataset.id+ for dataset in self.api.list_datasets(+ author=username, limit=limit, **kwargs+ )+ ]+ except Exception as e:+ logger.error(f"Error listing datasets: {e}")+ return []++ def delete_dataset(self, dataset_name: str, **kwargs: Any) -> None:+ r"""Deletes a dataset from the Hugging Face Hub.++ Args:+ dataset_name (str): The name of the dataset to delete.+ kwargs (Any): Additional keyword arguments.+ """+ try:+ self.api.delete_repo(+ repo_id=dataset_name,+ repo_type=HuggingFaceRepoType.DATASET.value,+ **kwargs,+ )+ logger.info(f"Dataset '{dataset_name}' deleted successfully.")+ except Exception as e:+ logger.error(f"Error deleting dataset '{dataset_name}': {e}")+ raise++ def add_records(+ self,+ dataset_name: str,+ records: List[Record],+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> None:+ r"""Adds records to a dataset on the Hugging Face Hub.++ Args:+ dataset_name (str): The name of the dataset.+ records (List[Record]): A list of records to add to the dataset.+ filepath (str): The path to the file containing the records.+ kwargs (Any): Additional keyword arguments.++ Raises:+ ValueError: If the dataset already has a records file.+ """+ existing_records = self._download_records(+ dataset_name=dataset_name, filepath=filepath, **kwargs+ )++ if existing_records:+ raise ValueError(+ f"Dataset '{filepath}' already exists. "+ f"Use `update_records` to modify."+ )++ self._upload_records(+ records=records,+ dataset_name=dataset_name,+ filepath=filepath,+ **kwargs,+ )++ def update_records(+ self,+ dataset_name: str,+ records: List[Record],+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> None:+ r"""Updates records in a dataset on the Hugging Face Hub.++ Args:+ dataset_name (str): The name of the dataset.+ records (List[Record]): A list of records to update in the dataset.+ filepath (str): The path to the file containing the records.+ kwargs (Any): Additional keyword arguments.++ Raises:+ ValueError: If the dataset does not have an existing file to update+ records in.+ """+ existing_records = self._download_records(+ dataset_name=dataset_name, filepath=filepath, **kwargs+ )++ if not existing_records:+ logger.warning(+ f"Dataset '{dataset_name}' does not have existing "+ "records. Adding new records."+ )+ self._upload_records(+ records=records,+ dataset_name=dataset_name,+ filepath=filepath,+ **kwargs,+ )+ return++ old_dict = {record.id: record for record in existing_records}+ new_dict = {record.id: record for record in records}+ merged_dict = old_dict.copy()+ merged_dict.update(new_dict)++ self._upload_records(+ records=list(merged_dict.values()),+ dataset_name=dataset_name,+ filepath=filepath,+ **kwargs,+ )++ def delete_record(+ self,+ dataset_name: str,+ record_id: str,+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> None:+ r"""Deletes a record from the dataset.++ Args:+ dataset_name (str): The name of the dataset.+ record_id (str): The ID of the record to delete.+ filepath (str): The path to the file containing the records.+ kwargs (Any): Additional keyword arguments.++ Raises:+ ValueError: If the dataset does not have an existing file to delete+ records from.+ """+ existing_records = self._download_records(+ dataset_name=dataset_name, filepath=filepath, **kwargs+ )++ if not existing_records:+ raise ValueError(+ f"Dataset '{dataset_name}' does not have an existing file to "+ f"delete records from."+ )++ filtered_records = [+ record for record in existing_records if record.id != record_id+ ]++ self._upload_records(+ records=filtered_records,+ dataset_name=dataset_name,+ filepath=filepath,+ **kwargs,+ )++ def list_records(+ self,+ dataset_name: str,+ filepath: str = "records/records.json",+ **kwargs: Any,+ ) -> List[Record]:+ r"""Lists all records in a dataset.++ Args:+ dataset_name (str): The name of the dataset.+ filepath (str): The path to the file containing the records.+ kwargs (Any): Additional keyword arguments.++ Returns:+ List[Record]: A list of records in the dataset.+ """+ return self._download_records(+ dataset_name=dataset_name, filepath=filepath, **kwargs+ )++ def _download_records(+ self, dataset_name: str, filepath: str, **kwargs: Any+ ) -> List[Record]:+ from huggingface_hub import hf_hub_download+ from huggingface_hub.errors import EntryNotFoundError++ try:+ downloaded_file_path = hf_hub_download(+ repo_id=dataset_name,+ filename=filepath,+ repo_type=HuggingFaceRepoType.DATASET.value,+ token=self._api_key,+ **kwargs,+ )++ with open(downloaded_file_path, "r") as f:+ records_data = json.load(f)++ return [Record(**record) for record in records_data]+ except EntryNotFoundError:+ logger.info(f"No records found for dataset '{dataset_name}'.")+ return []+ except Exception as e:+ logger.error(f"Error downloading or processing records: {e}")+ raise e++ def _upload_records(+ self,+ records: List[Record],+ dataset_name: str,+ filepath: str,+ **kwargs: Any,+ ):+ with tempfile.NamedTemporaryFile(+ delete=False, mode="w", newline="", encoding="utf-8"+ ) as f:+ json.dump([record.model_dump() for record in records], f)+ temp_file_path = f.name++ try:+ self.api.upload_file(+ path_or_fileobj=temp_file_path,+ path_in_repo=filepath,+ repo_id=dataset_name,+ repo_type=HuggingFaceRepoType.DATASET.value,+ **kwargs,+ )+ except Exception as e:+ logger.error(f"Error uploading records file: {e}")+ raise+ finally:+ if os.path.exists(temp_file_path):+ os.remove(temp_file_path)++ def _upload_file(+ self,+ file_content: str,+ dataset_name: str,+ filepath: str,+ file_type: str = "json",+ **kwargs: Any,+ ):+ with tempfile.NamedTemporaryFile(+ mode="w", delete=False, suffix=f".{file_type}"+ ) as f:+ if file_type == "json":+ if isinstance(file_content, str):+ try:+ json_content = json.loads(file_content)+ except json.JSONDecodeError:+ raise ValueError(+ "Invalid JSON string provided for file_content."+ )+ else:+ try:+ json.dumps(file_content)+ json_content = file_content+ except (TypeError, ValueError):+ raise ValueError(+ "file_content is not JSON serializable."+ )++ json.dump(json_content, f)+ elif file_type == "md" or file_type == "txt":+ f.write(file_content)+ else:+ raise ValueError(f"Unsupported file type: {file_type}")++ temp_file_path = f.name++ try:+ self.api.upload_file(+ path_or_fileobj=temp_file_path,+ path_in_repo=filepath,+ repo_id=dataset_name,+ repo_type=HuggingFaceRepoType.DATASET.value,+ **kwargs,+ )+ logger.info(f"File uploaded successfully: {filepath}")+ except Exception as e:+ logger.error(f"Error uploading file: {e}")+ raise++ if os.path.exists(temp_file_path):+ os.remove(temp_file_path)diff --git a/camel/datahubs/models.py b/camel/datahubs/models.pynew file mode 100644index 000000000..cd63de8f2--- /dev/null+++ b/camel/datahubs/models.py@@ -0,0 +1,22 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from typing import Any, Dict, Optional++from pydantic import BaseModel+++class Record(BaseModel):+ id: Optional[str] = None+ metadata: Optional[Dict[str, Any]] = None+ content: Dict[str, Any]diff --git a/camel/types/__init__.py b/camel/types/__init__.pyindex 7c6a3dcab..3904dc261 100644--- a/camel/types/__init__.py+++ b/camel/types/__init__.py@@ -14,6 +14,7 @@from .enums import (AudioModelType,EmbeddingModelType,+ HuggingFaceRepoType,ModelPlatformType,ModelType,OpenAIBackendRole,@@ -73,4 +74,5 @@ __all__ = ['NOT_GIVEN','NotGiven','ParsedChatCompletion',+ 'HuggingFaceRepoType',]diff --git a/camel/types/enums.py b/camel/types/enums.pyindex c623a683b..5cca71285 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -749,3 +749,9 @@ class JinaReturnFormat(Enum):MARKDOWN = "markdown"HTML = "html"TEXT = "text"+++class HuggingFaceRepoType(str, Enum):+ DATASET = "dataset"+ MODEL = "model"+ SPACE = "space"diff --git a/examples/datahubs/huggingface.py b/examples/datahubs/huggingface.pynew file mode 100644index 000000000..8f26e9137--- /dev/null+++ b/examples/datahubs/huggingface.py@@ -0,0 +1,117 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from camel.datahubs.huggingface import HuggingFaceDatasetManager+from camel.datahubs.models import Record++manager = HuggingFaceDatasetManager()++USERNAME = "username"+REPO_ID = f"{USERNAME}/test-dataset-example"++records = [+ Record(+ id="record-1",+ content={"input": "What is AI?", "output": "Artificial Intelligence"},+ metadata={"method": "SFT"},+ ),+ Record(+ id="record-2",+ content={"input": "Translate 'hello'", "output": "Bonjour"},+ metadata={"method": "GPT"},+ ),+]++# 1. create a dataset+print("Creating dataset...")+dataset_url = manager.create_dataset(name=REPO_ID)+print(f"Dataset created: {dataset_url}")++# 2. create a dataset card+print("Creating dataset card...")+manager.create_dataset_card(+ dataset_name=REPO_ID,+ description="Test dataset",+ license="mit",+ language=["en"],+ size_category="<1MB",+ version="0.1.0",+ tags=["test", "example"],+ task_categories=["other"],+ authors=["camel-ai"],+)+print("Dataset card created successfully.")++# 3. add records to the dataset+print("Adding records to the dataset...")+manager.add_records(dataset_name=REPO_ID, records=records)+print("Records added successfully.")++# 4. list all records+print("Listing all records...")+all_records = manager.list_records(dataset_name=REPO_ID)+print("Records in the dataset:")+for record in all_records:+ print(+ f"- ID: {record.id}, Input: {record.content['input']}, "+ f"Output: {record.content['output']}"+ )++# 5. update a record+new_records = [+ Record(+ id="record-3",+ content={"input": "What is ML?", "output": "Machine Learning"},+ metadata={"method": "Updated GPT"},+ )+]+print("Updating records...")+manager.update_records(dataset_name=REPO_ID, records=new_records)+print("Records updated successfully.")++# 6. list updated records+print("Listing updated records...")+updated_records = manager.list_records(dataset_name=REPO_ID)+print("Updated records in the dataset:")+for record in updated_records:+ print(+ f"- ID: {record.id}, Input: {record.content['input']}, "+ f"Output: {record.content['output']}"+ )++# 7. delete a record+print("Deleting record with ID 'record-1'...")+manager.delete_record(dataset_name=REPO_ID, record_id="record-1")+print("Record deleted successfully.")++# 8. list records after deletion+print("Listing records after deletion...")+final_records = manager.list_records(dataset_name=REPO_ID)+print("Final records in the dataset:")+for record in final_records:+ print(+ f"- ID: {record.id}, Input: {record.content['input']}, "+ f"Output: {record.content['output']}"+ )++# 9. list all datasets+print("Listing all datasets...")+datasets = manager.list_datasets(USERNAME)+print("Datasets:")+for dataset in datasets:+ print(f"- {dataset}")++# 10. delete a dataset+print(f"Deleting dataset: {REPO_ID}...")+manager.delete_dataset(dataset_name=REPO_ID)+print("Dataset deleted successfully.")diff --git a/poetry.lock b/poetry.lockindex 4da1bab0b..d66fd9acd 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.[[package]]name = "accelerate"@@ -2149,12 +2149,12 @@ files = [google-auth = ">=2.14.1,<3.0.dev0"googleapis-common-protos = ">=1.56.2,<2.0.dev0"grpcio = [- {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},{version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},+ {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},]grpcio-status = [- {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},{version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},+ {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""},]proto-plus = ">=1.22.3,<2.0.0dev"protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0"@@ -3916,6 +3916,7 @@ python-versions = ">=3.7"files = [{file = "milvus_lite-2.4.10-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:fc4246d3ed7d1910847afce0c9ba18212e93a6e9b8406048436940578dfad5cb"},{file = "milvus_lite-2.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:74a8e07c5e3b057df17fbb46913388e84df1dc403a200f4e423799a58184c800"},+ {file = "milvus_lite-2.4.10-py3-none-manylinux2014_aarch64.whl", hash = "sha256:240c7386b747bad696ecb5bd1f58d491e86b9d4b92dccee3315ed7256256eddc"},{file = "milvus_lite-2.4.10-py3-none-manylinux2014_x86_64.whl", hash = "sha256:211d2e334a043f9282bdd9755f76b9b2d93b23bffa7af240919ffce6a8dfe325"},]@@ -4882,9 +4883,9 @@ files = [[package.dependencies]numpy = [+ {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""},{version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""},{version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""},- {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""},{version = ">=1.26.0", markers = "python_version >= \"3.12\""},]@@ -5064,8 +5065,8 @@ files = [[package.dependencies]numpy = [- {version = ">=1.22.4", markers = "python_version < \"3.11\""},{version = ">=1.23.2", markers = "python_version == \"3.11\""},+ {version = ">=1.22.4", markers = "python_version < \"3.11\""},{version = ">=1.26.0", markers = "python_version >= \"3.12\""},]python-dateutil = ">=2.8.2"@@ -5910,6 +5911,7 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecsoptional = truepython-versions = ">=3.8"files = [+ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},{file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},]@@ -5920,6 +5922,7 @@ description = "A collection of ASN.1-based protocols modules"optional = truepython-versions = ">=3.8"files = [+ {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},{file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},]@@ -7357,40 +7360,30 @@ files = [{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a606ef75a60ecf3d924613892cc603b154178ee25abb3055db5062da811fd969"},{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd5415dded15c3822597455bc02bcd66e81ef8b7a48cb71a33628fc9fdde39df"},{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"},- {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"},- {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"},{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"},{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"},{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"},{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d84318609196d6bd6da0edfa25cedfbabd8dbde5140a0a23af29ad4b8f91fb1e"},{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb43a269eb827806502c7c8efb7ae7e9e9d0573257a46e8e952f4d4caba4f31e"},{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"},- {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"},- {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"},{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"},{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"},{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"},{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d"},{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c"},{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"},- {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"},- {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"},{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"},{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"},{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"},{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:e7e3736715fbf53e9be2a79eb4db68e4ed857017344d697e8b9749444ae57475"},{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7e75b4965e1d4690e93021adfcecccbca7d61c7bddd8e22406ef2ff20d74ef"},{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"},- {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"},- {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"},{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"},{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"},{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"},{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bc5f1e1c28e966d61d2519f2a3d451ba989f9ea0f2307de7bc45baa526de9e45"},{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a0e060aace4c24dcaf71023bbd7d42674e3b230f7e7b97317baf1e953e5b519"},{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"},- {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"},- {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"},{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"},{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"},{file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"},@@ -8893,6 +8886,17 @@ files = [{file = "types_python_dateutil-2.9.0.20241206.tar.gz", hash = "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb"},]+[[package]]+name = "types-pyyaml"+version = "6.0.12.20240917"+description = "Typing stubs for PyYAML"+optional = false+python-versions = ">=3.8"+files = [+ {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"},+ {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"},+]+[[package]]name = "types-regex"version = "2024.11.6.20241108"@@ -9885,7 +9889,7 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",type = ["pytest-mypy"][extras]-all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]+all = ["PyMuPDF", "accelerate", "agentops", "anthropic", "apify_client", "arxiv", "arxiv2text", "asknews", "azure-storage-blob", "beautifulsoup4", "botocore", "cohere", "cohere", "datacommons", "datacommons_pandas", "datasets", "diffusers", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "google-cloud-storage", "google-generativeai", "googlemaps", "imageio", "ipykernel", "jupyter_client", "litellm", "mistralai", "nebula3-python", "neo4j", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "opencv-python", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pymilvus", "pyowm", "pyyaml", "qdrant-client", "rank-bm25", "redis", "reka-api", "requests_oauthlib", "scholarly", "sentence-transformers", "sentencepiece", "slack-bolt", "slack-sdk", "soundfile", "tavily-python", "textblob", "torch", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]encoders = ["sentence-transformers"]graph-storages = ["nebula3-python", "neo4j"]huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "torch", "transformers"]@@ -9897,10 +9901,10 @@ retrievers = ["cohere", "rank-bm25"]runtime = ["docker"]search-tools = ["duckduckgo-search", "tavily-python", "wikipedia", "wolframalpha"]test = ["mock", "pytest", "pytest-asyncio"]-tools = ["PyMuPDF", "agentops", "apify_client", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "datacommons", "datacommons_pandas", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]+tools = ["PyMuPDF", "agentops", "apify_client", "arxiv", "arxiv2text", "asknews", "beautifulsoup4", "datacommons", "datacommons_pandas", "discord.py", "docker", "docx2txt", "duckduckgo-search", "ffmpeg-python", "firecrawl-py", "googlemaps", "imageio", "ipykernel", "jupyter_client", "newspaper3k", "nltk", "notion-client", "openapi-spec-validator", "pdfplumber", "pillow", "prance", "praw", "pyTelegramBotAPI", "pydub", "pygithub", "pyowm", "pyyaml", "requests_oauthlib", "scholarly", "slack-bolt", "slack-sdk", "textblob", "unstructured", "wikipedia", "wolframalpha", "yt-dlp"]vector-databases = ["pymilvus", "qdrant-client"][metadata]lock-version = "2.0"python-versions = ">=3.10,<3.13"-content-hash = "0f3fa1b2e436a1dafb85b426e0d863b6563e4aa9880f92be2af84e7946bd231d"+content-hash = "66d081b88fb592beaee2ce10058723920f82106a987f90490f6d75b746e43085"diff --git a/pyproject.toml b/pyproject.tomlindex d5bdd2763..04c740079 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -109,6 +109,7 @@ apify_client = { version = "^1.8.1", optional = true }notion-client = { version = "^2.2.1", optional = true }yt-dlp = { version = "^2024.11.4", optional = true }ffmpeg-python = { version = "^0.2.0", optional = true }+pyyaml = { version = "^6.0.2", optional = true }# encoders@@ -220,6 +221,7 @@ tools = ["notion-client","yt-dlp","ffmpeg-python",+ "pyyaml",]vector-databases = [@@ -301,6 +303,7 @@ all = ["notion-client","yt-dlp","ffmpeg-python",+ "pyyaml",# vector-database"qdrant-client","pymilvus",@@ -356,6 +359,7 @@ types-setuptools = "*"types-tqdm = "*"types-colorama = "^0"types-requests = "^2"+types-PyYAML = "^6"[tool.poetry.group.docs]optional = true@@ -430,6 +434,7 @@ module = ["database_connection","huggingface_hub","huggingface_hub.utils._errors",+ "huggingface_hub.errors","wikipedia","duckduckgo_search","newspaper",diff --git a/test/datahubs/test_huggingface.py b/test/datahubs/test_huggingface.pynew file mode 100644index 000000000..8e3a83f28--- /dev/null+++ b/test/datahubs/test_huggingface.py@@ -0,0 +1,296 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import datetime+import json+import os+from unittest.mock import MagicMock, patch++import pytest+from huggingface_hub.errors import EntryNotFoundError++from camel.datahubs.huggingface import HuggingFaceDatasetManager+from camel.datahubs.models import Record++TOKEN = "your_huggingface_token"+DATASET_NAME = (+ f"test-dataset-{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}"+)+REPO_ID = f"username/{DATASET_NAME}"++os.environ["HUGGING_FACE_TOKEN"] = TOKEN+++def manager():+ return HuggingFaceDatasetManager()+++def test_create_dataset(manager):+ with patch("huggingface_hub.HfApi.create_repo") as mock_create_repo:+ mock_create_repo.return_value = {"repo_id": REPO_ID}++ dataset_url = manager.create_dataset(name=REPO_ID)+ assert dataset_url == f"https://huggingface.co/datasets/{REPO_ID}"++ mock_create_repo.assert_called_once_with(+ repo_id=REPO_ID, repo_type="dataset", private=False+ )+++def test_create_dataset_card():+ manager = HuggingFaceDatasetManager(token="fake-token")+ dataset_name = "user/test-dataset"+ description = "Test dataset"+ license = "MIT"+ version = "0.1.0"+ tags = ["test", "example"]+ authors = ["camel-ai"]+ size_category = "<1MB"+ language = ["en"]+ task_categories = ["other"]+ content = "Additional information about the dataset."++ with patch(+ "camel.datahubs.HuggingFaceDatasetManager._upload_file"+ ) as mock_upload_file:+ mock_upload_file.return_value = None++ manager.create_dataset_card(+ dataset_name=dataset_name,+ description=description,+ license=license,+ version=version,+ tags=tags,+ authors=authors,+ size_category=size_category,+ language=language,+ task_categories=task_categories,+ content=content,+ )++ mock_upload_file.assert_called_once_with(+ file_content="---\nauthors:\n- camel-ai\ndescription: Test dataset"+ "\nlanguage:\n- en\nlicense: MIT\npretty_name: "+ "user/test-dataset\nsize_categories: <1MB\ntags:\n"+ "- test\n- example\ntask_categories:\n- other\n"+ "version: 0.1.0\n\n---\n\n# Additional Information\n"+ "Additional information about the dataset.\n",+ dataset_name=dataset_name,+ filepath="README.md",+ file_type="md",+ )+++def test_add_records(manager):+ records = [+ Record(+ id="record-1",+ content={+ "input": "What is AI?",+ "output": "Artificial Intelligence",+ },+ metadata={"method": "SFT"},+ ),+ Record(+ id="record-2",+ content={"input": "Translate 'hello'", "output": "Bonjour"},+ metadata={"method": "GPT"},+ ),+ ]++ with (+ patch(+ "huggingface_hub.hf_hub_download",+ side_effect=EntryNotFoundError("404 Client Error."),+ ) as mock_download,+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,+ ):+ manager.add_records(dataset_name=REPO_ID, records=records)++ mock_download.assert_called_once()+ mock_upload_file.assert_called_once()+ call_args = mock_upload_file.call_args[1]+ assert call_args["repo_id"] == REPO_ID+ assert call_args["path_in_repo"] == "records/records.json"+ assert call_args["repo_type"] == "dataset"+++def test_update_records(manager):+ existing_records = [+ {+ "id": "record-1",+ "content": {+ "input": "What is AI?",+ "output": "Artificial Intelligence",+ },+ "metadata": {"method": "SFT"},+ }+ ]++ new_records = [+ Record(+ id="record-2",+ content={"input": "Translate 'hello'", "output": "Salut"},+ metadata={"method": "Updated GPT"},+ ),+ Record(+ id="record-3",+ content={"input": "What is ML?", "output": "Machine Learning"},+ metadata={"method": "SFT"},+ ),+ ]++ with (+ patch("huggingface_hub.hf_hub_download") as mock_download,+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,+ patch("builtins.open", new_callable=MagicMock) as mock_open,+ ):+ mock_download.return_value = "/mock/path/records.json"++ mock_open.return_value.__enter__.return_value.read.return_value = (+ json.dumps(existing_records)+ )++ manager.update_records(dataset_name=REPO_ID, records=new_records)++ mock_upload_file.assert_called_once()+ call_args = mock_upload_file.call_args[1]+ assert call_args["repo_id"] == REPO_ID+ assert call_args["path_in_repo"] == "records/records.json"+ assert call_args["repo_type"] == "dataset"+++def test_list_records(manager):+ mock_records = [+ {+ "id": "record-1",+ "content": {+ "input": "What is AI?",+ "output": "Artificial Intelligence",+ },+ "metadata": {"method": "SFT"},+ },+ {+ "id": "record-2",+ "content": {"input": "Translate 'hello'", "output": "Salut"},+ "metadata": {"method": "GPT"},+ },+ ]++ with (+ patch("huggingface_hub.hf_hub_download") as mock_download,+ patch("builtins.open", new_callable=MagicMock) as mock_open,+ ):+ mock_download.return_value = "/mock/path/records.json"+ mock_open.return_value.__enter__.return_value.read.return_value = (+ json.dumps(mock_records)+ )++ records = manager.list_records(dataset_name=REPO_ID)+ assert len(records) == len(mock_records)++ for original, loaded in zip(mock_records, records):+ assert original["id"] == loaded.id+ assert original["content"] == loaded.content+ assert original["metadata"] == loaded.metadata+++def test_add_records_error_on_existing_file(manager):+ records = [+ Record(+ id="record-1",+ content={+ "input": "What is AI?",+ "output": "Artificial Intelligence",+ },+ metadata={"method": "SFT"},+ )+ ]++ with (+ patch(+ "huggingface_hub.hf_hub_download",+ side_effect=ValueError(+ "Dataset 'records/records.json' already exists. "+ "Use `update_records` to modify."+ ),+ ) as mock_download,+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,+ ):+ mock_download.return_value = "/mock/path/records.json"++ with pytest.raises(+ ValueError,+ match="Dataset 'records/records.json' already exists. "+ "Use `update_records` to modify.",+ ):+ manager.add_records(dataset_name=REPO_ID, records=records)++ mock_upload_file.assert_not_called()+++def test_list_datasets(manager):+ with patch("huggingface_hub.HfApi.list_datasets") as mock_list_datasets:+ mock_dataset = MagicMock()+ mock_dataset.id = REPO_ID+ mock_list_datasets.return_value = [mock_dataset]++ datasets = manager.list_datasets(username="username")+ assert datasets == [REPO_ID]++ mock_list_datasets.assert_called_once()+++def test_delete_dataset(manager):+ with patch("huggingface_hub.HfApi.delete_repo") as mock_delete_repo:+ manager.delete_dataset(dataset_name=REPO_ID)+ mock_delete_repo.assert_called_once_with(+ repo_id=REPO_ID, repo_type="dataset"+ )+++def test_delete_record(manager):+ with (+ patch("huggingface_hub.hf_hub_download") as mock_download,+ patch("huggingface_hub.HfApi.upload_file") as mock_upload_file,+ patch("builtins.open", new_callable=MagicMock) as mock_open,+ ):+ mock_download.return_value = "/mock/path/records.json"++ existing_records = [+ Record(+ id="record-1",+ content={+ "input": "What is AI?",+ "output": "Artificial Intelligence",+ },+ metadata={"method": "SFT"},+ ),+ Record(+ id="record-2",+ content={"input": "Translate 'hello'", "output": "Bonjour"},+ metadata={"method": "GPT"},+ ),+ ]+ mock_open.return_value.__enter__.return_value.read.return_value = (+ json.dumps([record.model_dump() for record in existing_records])+ )++ manager.delete_record(dataset_name=REPO_ID, record_id="record-1")++ mock_upload_file.assert_called_once()+ call_args = mock_upload_file.call_args[1]+ assert call_args["repo_id"] == REPO_ID+ assert call_args["path_in_repo"] == "records/records.json"+ assert call_args["repo_type"] == "dataset"