🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming
allow datasets to be prepared by users
Yizhou Chi07800be4417c
diff --git a/expo/data/dataset.py b/expo/data/dataset.pyindex 8b0c5b980..91490dcd7 100644--- a/expo/data/dataset.py+++ b/expo/data/dataset.py@@ -1,3 +1,4 @@+import argparseimport asyncioimport jsonimport os@@ -18,22 +19,22 @@ Report {metric} on the eval data. Do not plot or make any visualizations."""USE_AG = """-7. Please use autogluon for model training with presets='medium_quality', time_limit=None, give dev dataset to tuning_data, and use right eval_metric.+- Please use autogluon for model training with presets='medium_quality', time_limit=None, give dev dataset to tuning_data, and use right eval_metric."""TEXT_MODALITY = """-7. You could use models from transformers library for this text dataset.-8. Use gpu if available for faster training.+- You could use models from transformers library for this text dataset.+- Use gpu if available for faster training."""IMAGE_MODALITY = """-7. You could use models from transformers/torchvision library for this image dataset.-8. Use gpu if available for faster training.+- You could use models from transformers/torchvision library for this image dataset.+- Use gpu if available for faster training."""STACKING = """-7. To avoid overfitting, train a weighted ensemble model such as StackingClassifier or StackingRegressor.-8. You could do some quick model prototyping to see which models work best and then use them in the ensemble.+- To avoid overfitting, train a weighted ensemble model such as StackingClassifier or StackingRegressor.+- You could do some quick model prototyping to see which models work best and then use them in the ensemble."""@@ -361,10 +362,22 @@ async def process_dataset(dataset, solution_designer: SolutionDesigner, save_anadatasets_dict["datasets"][dataset.name] = dataset_dict+def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("--force_update", action="store_true", help="Force update datasets")+ parser.add_argument("--save_analysis_pool", action="store_true", help="Save analysis pool")+ parser.add_argument(+ "--no_save_analysis_pool", dest="save_analysis_pool", action="store_false", help="Do not save analysis pool"+ )+ parser.set_defaults(save_analysis_pool=True)+ return parser.parse_args()++if __name__ == "__main__":datasets_dir = DATA_CONFIG["datasets_dir"]- force_update = False- save_analysis_pool = True+ args = parse_args()+ force_update = args.force_update+ save_analysis_pool = args.save_analysis_pooldatasets_dict = {"datasets": {}}solution_designer = SolutionDesigner()for dataset_id in OPENML_DATASET_IDS:diff --git a/expo/data/hf_data.py b/expo/data/hf_data.pyindex 133fbdfa6..a18517d49 100644--- a/expo/data/hf_data.py+++ b/expo/data/hf_data.py@@ -7,7 +7,12 @@ import pandas as pdfrom datasets import load_datasetfrom PIL import Image-from expo.data.dataset import ExpDataset, process_dataset, save_datasets_dict_to_yaml+from expo.data.dataset import (+ ExpDataset,+ parse_args,+ process_dataset,+ save_datasets_dict_to_yaml,+)from expo.insights.solution_designer import SolutionDesignerfrom expo.utils import DATA_CONFIG@@ -116,8 +121,9 @@ class HFExpDataset(ExpDataset):if __name__ == "__main__":dataset_dir = DATA_CONFIG["datasets_dir"]- save_analysis_pool = True- force_update = False+ args = parse_args()+ force_update = args.force_update+ save_analysis_pool = args.save_analysis_pooldatasets_dict = {"datasets": {}}solution_designer = SolutionDesigner()for dataset_meta in HFDATSETS: