Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Save_function for loong (#2509)

Yifeng Wang(正经人王同学)490e29bbddee
diff --git a/camel/datasets/static_dataset.py b/camel/datasets/static_dataset.py
index 9ff726746..3ac8519f4 100644
--- a/camel/datasets/static_dataset.py
+++ b/camel/datasets/static_dataset.py
@@ -398,3 +398,147 @@ class StaticDataset(Dataset):
f"got {type(item).__name__}"
)
return data
+
+ def save_to_json(self, file_path: Union[str, Path]) -> None:
+ r"""Save the dataset to a local JSON file.
+
+ Args:
+ file_path (Union[str, Path]): Path to the output JSON file.
+ If a string is provided, it will be converted to a Path object.
+
+ Raises:
+ TypeError: If file_path is not a string or Path object.
+ OSError: If there's an error writing to the file.
+ """
+ if isinstance(file_path, str):
+ file_path = Path(file_path)
+ elif not isinstance(file_path, Path):
+ raise TypeError(
+ f"Expected file_path to be a string or Path object, "
+ f"got {type(file_path).__name__}"
+ )
+
+ # Convert DataPoint objects to dictionaries
+ data_dicts = [datapoint.to_dict() for datapoint in self.data]
+
+ # Ensure the parent directory exists
+ file_path.parent.mkdir(parents=True, exist_ok=True)
+
+ try:
+ logger.debug(f"Saving dataset to {file_path}")
+ with file_path.open('w', encoding='utf-8') as f:
+ json.dump(data_dicts, f, ensure_ascii=False, indent=2)
+ logger.info(
+ f"Successfully saved {len(data_dicts)} items to {file_path}"
+ )
+ except OSError as e:
+ logger.error(f"Error saving dataset to {file_path}: {e}")
+ raise
+
+ def save_to_huggingface(
+ self,
+ dataset_name: str,
+ token: Optional[str] = None,
+ filepath: str = "records/records.json",
+ private: bool = False,
+ description: Optional[str] = None,
+ license: Optional[str] = None,
+ version: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ language: Optional[List[str]] = None,
+ task_categories: Optional[List[str]] = None,
+ authors: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> str:
+ r"""Save the dataset to the Hugging Face Hub using the project's
+ HuggingFaceDatasetManager.
+
+ Args:
+ dataset_name (str): The name of the dataset on Hugging Face Hub.
+ Should be in the format 'username/dataset_name' .
+ token (Optional[str]): The Hugging Face API token. If not provided,
+ the token will be read from the environment variable `HF_TOKEN`
+ (default: :obj:`None`)
+ filepath (str): The path in the repository where the dataset
+ will be saved. (default: :obj:`"records/records.json"`)
+ private (bool): Whether the dataset should be private.
+ (default: :obj:`False`)
+ description (Optional[str]): A description of the dataset.
+ (default: :obj:`None`)
+ license (Optional[str]): The license of the dataset.
+ (default: :obj:`None`)
+ version (Optional[str]): The version of the dataset.
+ (default: :obj:`None`)
+ tags (Optional[List[str]]): A list of tags for the dataset.
+ (default: :obj:`None`)
+ language (Optional[List[str]]): A list of languages the dataset is
+ in. (default: :obj:`None`)
+ task_categories (Optional[List[str]]): A list of task categories.
+ (default: :obj:`None`)
+ authors (Optional[List[str]]): A list of authors of the dataset.
+ (default: :obj:`None`)
+ **kwargs (Any): Additional keyword arguments to pass to the
+ Hugging Face API.
+
+ Returns:
+ str: The URL of the dataset on the Hugging Face Hub.
+
+ Raises:
+ OSError: If there's an error uploading the dataset.
+ """
+ # lazy import to avoid heavy dependencies
+ from camel.datahubs.huggingface import HuggingFaceDatasetManager
+ from camel.datahubs.models import Record
+
+ # Initialize the HuggingFaceDatasetManager
+ manager = HuggingFaceDatasetManager(token=token)
+
+ # Convert DataPoint objects to Record objects
+ records = []
+ for datapoint in self.data:
+ datapoint_dict = datapoint.to_dict()
+
+ record_dict = {
+ "question": datapoint_dict.get("question", ""),
+ "final_answer": datapoint_dict.get("final_answer", ""),
+ "rationale": datapoint_dict.get("rationale", ""),
+ "metadata": datapoint_dict.get("metadata", {}),
+ }
+ record = Record(**record_dict)
+ records.append(record)
+
+ logger.debug(f"Creating dataset {dataset_name}")
+ try:
+ # Create the dataset
+ dataset_url = manager.create_dataset(
+ name=dataset_name, private=private, **kwargs
+ )
+
+ # Add records to the dataset
+ manager.add_records(
+ dataset_name=dataset_name,
+ records=records,
+ filepath=filepath,
+ )
+
+ # Create dataset card if description is provided
+ if description:
+ manager.create_dataset_card(
+ dataset_name=dataset_name,
+ description=description,
+ license=license,
+ version=version,
+ tags=tags,
+ authors=authors,
+ language=language,
+ task_categories=task_categories,
+ )
+
+ logger.info(
+ f"Successfully uploaded dataset to {dataset_name}, "
+ f"the url is {dataset_url}"
+ )
+ return dataset_url
+ except Exception as e:
+ logger.error(f"Error uploading dataset to Hugging Face: {e}")
+ raise
diff --git a/docs/mintlify/reference/camel.datasets.static_dataset.mdx b/docs/mintlify/reference/camel.datasets.static_dataset.mdx
index eae843ef3..73bd3ee2a 100644
--- a/docs/mintlify/reference/camel.datasets.static_dataset.mdx
+++ b/docs/mintlify/reference/camel.datasets.static_dataset.mdx
@@ -207,3 +207,61 @@ Validate and convert a list of dictionaries into a dataset.
**Returns:**
List[Dict[str, Any]]: The validated list of dictionaries.
+
+<a id="camel.datasets.static_dataset.StaticDataset.save_to_json"></a>
+
+### save_to_json
+
+```python
+def save_to_json(self, file_path: Union[str, Path]):
+```
+
+Save the dataset to a local JSON file.
+
+**Parameters:**
+
+- **file_path** (Union[str, Path]): Path to the output JSON file. If a string is provided, it will be converted to a Path object.
+
+<a id="camel.datasets.static_dataset.StaticDataset.save_to_huggingface"></a>
+
+### save_to_huggingface
+
+```python
+def save_to_huggingface(
+ self,
+ dataset_name: str,
+ token: Optional[str] = None,
+ filepath: str = 'records/records.json',
+ private: bool = False,
+ description: Optional[str] = None,
+ license: Optional[str] = None,
+ version: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ language: Optional[List[str]] = None,
+ task_categories: Optional[List[str]] = None,
+ authors: Optional[List[str]] = None,
+ overwrite: bool = False,
+ **kwargs: Any
+):
+```
+
+Save the dataset to the Hugging Face Hub using the project's
+HuggingFaceDatasetManager.
+
+**Parameters:**
+
+- **dataset_name** (str): The name of the dataset on Hugging Face Hub. Should be in the format 'username/dataset_name' .
+- **token** (Optional[str]): The Hugging Face API token. If not provided, the token will be read from the environment variable `HF_TOKEN` (default: :obj:`None`)
+- **filepath** (str): The path in the repository where the dataset will be saved. (default: :obj:`"records/records.json"`)
+- **private** (bool): Whether the dataset should be private. (default: :obj:`False`)
+- **description** (Optional[str]): A description of the dataset. (default: :obj:`None`)
+- **license** (Optional[str]): The license of the dataset. (default: :obj:`None`)
+- **version** (Optional[str]): The version of the dataset. (default: :obj:`None`)
+- **tags** (Optional[List[str]]): A list of tags for the dataset. (default: :obj:`None`)
+- **language** (Optional[List[str]]): A list of languages the dataset is in. (default: :obj:`None`)
+- **task_categories** (Optional[List[str]]): A list of task categories. (default: :obj:`None`)
+- **authors** (Optional[List[str]]): A list of authors of the dataset. (default: :obj:`None`) **kwargs (Any): Additional keyword arguments to pass to the Hugging Face API.
+
+**Returns:**
+
+ str: The URL of the dataset on the Hugging Face Hub.
diff --git a/examples/dataset/static_data_saving.py b/examples/dataset/static_data_saving.py
new file mode 100644
index 000000000..77c404146
--- /dev/null
+++ b/examples/dataset/static_data_saving.py
@@ -0,0 +1,75 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from pathlib import Path
+
+from camel.datasets import StaticDataset
+
+raw_data2 = [
+ {
+ "question": "Evaluate the limit as x approaches 0 of (sin(3*x) - 3*x) / x**3.", # noqa: E501
+ "final_answer": "-9/2",
+ "rationale": '''from sympy import symbols, limit, sin
+x = symbols('x')
+expr = (sin(3*x) - 3*x) / x**3
+result = limit(expr, x, 0)
+print(result)''',
+ },
+ {
+ "question": "Evaluate the definite integral of (1 - x**2)**3 from x = 0 to x = 1.", # noqa: E501
+ "final_answer": "16/35",
+ "rationale": '''from sympy import symbols, integrate
+x = symbols('x')
+expr = (1 - x**2)**3
+result = integrate(expr, (x, 0, 1))
+print(result)''',
+ },
+ {
+ "question": "Evaluate the limit as n approaches infinity of n*(sqrt(n**2 + 1) - n).", # noqa: E501
+ "final_answer": "1/2",
+ "rationale": '''from sympy import symbols, limit, sqrt
+n = symbols('n', positive=True)
+expr = n*(sqrt(n**2 + 1) - n)
+result = limit(expr, n, float("inf"))
+print(result)''',
+ },
+ {
+ "question": "Compute the sum of the series sum from n = 1 to 50 of 1/(n*(n+1)).", # noqa: E501
+ "final_answer": "50/51",
+ "rationale": '''from sympy import symbols, summation
+n = symbols('n', positive=True, integer=True)
+expr = 1/(n*(n+1))
+result = summation(expr, (n, 1, 50))
+print(result)''',
+ },
+]
+
+seed_dataset = StaticDataset(raw_data2)
+
+
+# Save to JSON file
+seed_dataset.save_to_json("./localdata/output.json")
+# or
+seed_dataset.save_to_json(Path("./localdata/output.json"))
+
+
+# Save to Hugging Face Hub (first time)
+seed_dataset.save_to_huggingface(
+ dataset_name="zjrwtxtechstudio/testloongdata04",
+ description="A dataset containing question-answer pairs",
+ private=False,
+ tags=["LOONG_TEST", "qa"],
+ language=["en"],
+ overwrite=True,
+)