Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Autodownload tasks (#158)

Hasan Abed Al Kader Hammoud8f934383d0ae
diff --git a/camel/utils.py b/camel/utils.py
index d66c84a66..e379a1999 100644
--- a/camel/utils.py
+++ b/camel/utils.py
@@ -14,13 +14,15 @@
import os
import re
import time
+import zipfile
from functools import wraps
from typing import Any, Callable, List, Optional, Set, TypeVar
+import requests
import tiktoken
from camel.messages import OpenAIMessage
-from camel.typing import ModelType
+from camel.typing import ModelType, TaskType
F = TypeVar('F', bound=Callable[..., Any])
@@ -186,3 +188,22 @@ def get_first_int(string: str) -> Optional[int]:
return int(match.group())
else:
return None
+
+
+def download_tasks(task: TaskType, folder_path: str) -> None:
+ # Define the path to save the zip file
+ zip_file_path = os.path.join(folder_path, "tasks.zip")
+
+ # Download the zip file from the Google Drive link
+ response = requests.get("https://huggingface.co/datasets/camel-ai/"
+ f"metadata/resolve/main/{task.value}_tasks.zip")
+
+ # Save the zip file
+ with open(zip_file_path, "wb") as f:
+ f.write(response.content)
+
+ with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
+ zip_ref.extractall(folder_path)
+
+ # Delete the zip file
+ os.remove(zip_file_path)
diff --git a/examples/ai_society/role_playing_multiprocess.py b/examples/ai_society/role_playing_multiprocess.py
index 773f91e9c..2bdf391b0 100644
--- a/examples/ai_society/role_playing_multiprocess.py
+++ b/examples/ai_society/role_playing_multiprocess.py
@@ -20,6 +20,8 @@ from colorama import Fore
from camel.agents import RolePlaying
from camel.configs import ChatGPTConfig
+from camel.typing import TaskType
+from camel.utils import download_tasks
def generate_data(assistant_idx: int, assistant_role_name: str, user_idx: int,
@@ -181,6 +183,18 @@ def main() -> None:
# Disable/Enable Printing
verbose = True
+ # Check for tasks folder and install if not exists
+ # Define the folder path
+ folder_path = "./ai_society_data/"
+
+ # Check if the folder already exists
+ if not os.path.exists(folder_path):
+ os.makedirs(folder_path)
+
+ # Check if the folder is empty
+ if not os.listdir(folder_path):
+ download_tasks(task=TaskType.AI_SOCIETY, folder_path=folder_path)
+
# Chunk for parallel jobs
try:
array_idx = int(os.environ.get('SLURM_ARRAY_TASK_ID'))
diff --git a/examples/code/role_playing_multiprocess.py b/examples/code/role_playing_multiprocess.py
index 55192838f..155ca141d 100644
--- a/examples/code/role_playing_multiprocess.py
+++ b/examples/code/role_playing_multiprocess.py
@@ -25,6 +25,7 @@ from camel.messages import (
UserSystemMessage,
)
from camel.typing import RoleType, TaskType
+from camel.utils import download_tasks
def init_chat(
@@ -209,8 +210,24 @@ def generate_data(language_idx: int, language_name: str, domain_idx: int,
def main() -> None:
+ # Define the folder path
+ folder_path = "./code_data/"
+
+ # Check if the folder already exists
+ if not os.path.exists(folder_path):
+ os.makedirs(folder_path)
+
+ # Check if the folder is empty
+ if not os.listdir(folder_path):
+ download_tasks(task=TaskType.CODE, folder_path=folder_path)
+
# Chunk for parallel jobs
- array_idx = int(os.environ.get('SLURM_ARRAY_TASK_ID'))
+ try:
+ array_idx = int(os.environ.get('SLURM_ARRAY_TASK_ID'))
+ except (TypeError, ValueError) as e:
+ print(f"Error: {e}")
+ array_idx = 0
+
languages_per_chunk = 4
# Parameters for filtering the generated task string