🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Save_function for loong (#2509)
Yifeng Wang(正经人王同学)490e29bbddee
diff --git a/camel/datasets/static_dataset.py b/camel/datasets/static_dataset.pyindex 9ff726746..3ac8519f4 100644--- a/camel/datasets/static_dataset.py+++ b/camel/datasets/static_dataset.py@@ -398,3 +398,147 @@ class StaticDataset(Dataset):f"got {type(item).__name__}")return data++ def save_to_json(self, file_path: Union[str, Path]) -> None:+ r"""Save the dataset to a local JSON file.++ Args:+ file_path (Union[str, Path]): Path to the output JSON file.+ If a string is provided, it will be converted to a Path object.++ Raises:+ TypeError: If file_path is not a string or Path object.+ OSError: If there's an error writing to the file.+ """+ if isinstance(file_path, str):+ file_path = Path(file_path)+ elif not isinstance(file_path, Path):+ raise TypeError(+ f"Expected file_path to be a string or Path object, "+ f"got {type(file_path).__name__}"+ )++ # Convert DataPoint objects to dictionaries+ data_dicts = [datapoint.to_dict() for datapoint in self.data]++ # Ensure the parent directory exists+ file_path.parent.mkdir(parents=True, exist_ok=True)++ try:+ logger.debug(f"Saving dataset to {file_path}")+ with file_path.open('w', encoding='utf-8') as f:+ json.dump(data_dicts, f, ensure_ascii=False, indent=2)+ logger.info(+ f"Successfully saved {len(data_dicts)} items to {file_path}"+ )+ except OSError as e:+ logger.error(f"Error saving dataset to {file_path}: {e}")+ raise++ def save_to_huggingface(+ self,+ dataset_name: str,+ token: Optional[str] = None,+ filepath: str = "records/records.json",+ private: bool = False,+ description: Optional[str] = None,+ license: Optional[str] = None,+ version: Optional[str] = None,+ tags: Optional[List[str]] = None,+ language: Optional[List[str]] = None,+ task_categories: Optional[List[str]] = None,+ authors: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> str:+ r"""Save the dataset to the Hugging Face Hub using the project's+ HuggingFaceDatasetManager.++ Args:+ dataset_name (str): The name of the dataset on Hugging Face Hub.+ Should be in the format 'username/dataset_name' .+ token (Optional[str]): The Hugging Face API token. If not provided,+ the token will be read from the environment variable `HF_TOKEN`+ (default: :obj:`None`)+ filepath (str): The path in the repository where the dataset+ will be saved. (default: :obj:`"records/records.json"`)+ private (bool): Whether the dataset should be private.+ (default: :obj:`False`)+ description (Optional[str]): A description of the dataset.+ (default: :obj:`None`)+ license (Optional[str]): The license of the dataset.+ (default: :obj:`None`)+ version (Optional[str]): The version of the dataset.+ (default: :obj:`None`)+ tags (Optional[List[str]]): A list of tags for the dataset.+ (default: :obj:`None`)+ language (Optional[List[str]]): A list of languages the dataset is+ in. (default: :obj:`None`)+ task_categories (Optional[List[str]]): A list of task categories.+ (default: :obj:`None`)+ authors (Optional[List[str]]): A list of authors of the dataset.+ (default: :obj:`None`)+ **kwargs (Any): Additional keyword arguments to pass to the+ Hugging Face API.++ Returns:+ str: The URL of the dataset on the Hugging Face Hub.++ Raises:+ OSError: If there's an error uploading the dataset.+ """+ # lazy import to avoid heavy dependencies+ from camel.datahubs.huggingface import HuggingFaceDatasetManager+ from camel.datahubs.models import Record++ # Initialize the HuggingFaceDatasetManager+ manager = HuggingFaceDatasetManager(token=token)++ # Convert DataPoint objects to Record objects+ records = []+ for datapoint in self.data:+ datapoint_dict = datapoint.to_dict()++ record_dict = {+ "question": datapoint_dict.get("question", ""),+ "final_answer": datapoint_dict.get("final_answer", ""),+ "rationale": datapoint_dict.get("rationale", ""),+ "metadata": datapoint_dict.get("metadata", {}),+ }+ record = Record(**record_dict)+ records.append(record)++ logger.debug(f"Creating dataset {dataset_name}")+ try:+ # Create the dataset+ dataset_url = manager.create_dataset(+ name=dataset_name, private=private, **kwargs+ )++ # Add records to the dataset+ manager.add_records(+ dataset_name=dataset_name,+ records=records,+ filepath=filepath,+ )++ # Create dataset card if description is provided+ if description:+ manager.create_dataset_card(+ dataset_name=dataset_name,+ description=description,+ license=license,+ version=version,+ tags=tags,+ authors=authors,+ language=language,+ task_categories=task_categories,+ )++ logger.info(+ f"Successfully uploaded dataset to {dataset_name}, "+ f"the url is {dataset_url}"+ )+ return dataset_url+ except Exception as e:+ logger.error(f"Error uploading dataset to Hugging Face: {e}")+ raisediff --git a/docs/mintlify/reference/camel.datasets.static_dataset.mdx b/docs/mintlify/reference/camel.datasets.static_dataset.mdxindex eae843ef3..73bd3ee2a 100644--- a/docs/mintlify/reference/camel.datasets.static_dataset.mdx+++ b/docs/mintlify/reference/camel.datasets.static_dataset.mdx@@ -207,3 +207,61 @@ Validate and convert a list of dictionaries into a dataset.**Returns:**List[Dict[str, Any]]: The validated list of dictionaries.++<a id="camel.datasets.static_dataset.StaticDataset.save_to_json"></a>++### save_to_json++```python+def save_to_json(self, file_path: Union[str, Path]):+```++Save the dataset to a local JSON file.++**Parameters:**++- **file_path** (Union[str, Path]): Path to the output JSON file. If a string is provided, it will be converted to a Path object.++<a id="camel.datasets.static_dataset.StaticDataset.save_to_huggingface"></a>++### save_to_huggingface++```python+def save_to_huggingface(+ self,+ dataset_name: str,+ token: Optional[str] = None,+ filepath: str = 'records/records.json',+ private: bool = False,+ description: Optional[str] = None,+ license: Optional[str] = None,+ version: Optional[str] = None,+ tags: Optional[List[str]] = None,+ language: Optional[List[str]] = None,+ task_categories: Optional[List[str]] = None,+ authors: Optional[List[str]] = None,+ overwrite: bool = False,+ **kwargs: Any+):+```++Save the dataset to the Hugging Face Hub using the project's+HuggingFaceDatasetManager.++**Parameters:**++- **dataset_name** (str): The name of the dataset on Hugging Face Hub. Should be in the format 'username/dataset_name' .+- **token** (Optional[str]): The Hugging Face API token. If not provided, the token will be read from the environment variable `HF_TOKEN` (default: :obj:`None`)+- **filepath** (str): The path in the repository where the dataset will be saved. (default: :obj:`"records/records.json"`)+- **private** (bool): Whether the dataset should be private. (default: :obj:`False`)+- **description** (Optional[str]): A description of the dataset. (default: :obj:`None`)+- **license** (Optional[str]): The license of the dataset. (default: :obj:`None`)+- **version** (Optional[str]): The version of the dataset. (default: :obj:`None`)+- **tags** (Optional[List[str]]): A list of tags for the dataset. (default: :obj:`None`)+- **language** (Optional[List[str]]): A list of languages the dataset is in. (default: :obj:`None`)+- **task_categories** (Optional[List[str]]): A list of task categories. (default: :obj:`None`)+- **authors** (Optional[List[str]]): A list of authors of the dataset. (default: :obj:`None`) **kwargs (Any): Additional keyword arguments to pass to the Hugging Face API.++**Returns:**++ str: The URL of the dataset on the Hugging Face Hub.diff --git a/examples/dataset/static_data_saving.py b/examples/dataset/static_data_saving.pynew file mode 100644index 000000000..77c404146--- /dev/null+++ b/examples/dataset/static_data_saving.py@@ -0,0 +1,75 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from pathlib import Path++from camel.datasets import StaticDataset++raw_data2 = [+ {+ "question": "Evaluate the limit as x approaches 0 of (sin(3*x) - 3*x) / x**3.", # noqa: E501+ "final_answer": "-9/2",+ "rationale": '''from sympy import symbols, limit, sin+x = symbols('x')+expr = (sin(3*x) - 3*x) / x**3+result = limit(expr, x, 0)+print(result)''',+ },+ {+ "question": "Evaluate the definite integral of (1 - x**2)**3 from x = 0 to x = 1.", # noqa: E501+ "final_answer": "16/35",+ "rationale": '''from sympy import symbols, integrate+x = symbols('x')+expr = (1 - x**2)**3+result = integrate(expr, (x, 0, 1))+print(result)''',+ },+ {+ "question": "Evaluate the limit as n approaches infinity of n*(sqrt(n**2 + 1) - n).", # noqa: E501+ "final_answer": "1/2",+ "rationale": '''from sympy import symbols, limit, sqrt+n = symbols('n', positive=True)+expr = n*(sqrt(n**2 + 1) - n)+result = limit(expr, n, float("inf"))+print(result)''',+ },+ {+ "question": "Compute the sum of the series sum from n = 1 to 50 of 1/(n*(n+1)).", # noqa: E501+ "final_answer": "50/51",+ "rationale": '''from sympy import symbols, summation+n = symbols('n', positive=True, integer=True)+expr = 1/(n*(n+1))+result = summation(expr, (n, 1, 50))+print(result)''',+ },+]++seed_dataset = StaticDataset(raw_data2)+++# Save to JSON file+seed_dataset.save_to_json("./localdata/output.json")+# or+seed_dataset.save_to_json(Path("./localdata/output.json"))+++# Save to Hugging Face Hub (first time)+seed_dataset.save_to_huggingface(+ dataset_name="zjrwtxtechstudio/testloongdata04",+ description="A dataset containing question-answer pairs",+ private=False,+ tags=["LOONG_TEST", "qa"],+ language=["en"],+ overwrite=True,+)