Skip to content

A programming framework for agentic AI

when max_iter=1, skip search only if retrain_final (#280)

Chi Wang92ebd1f7f9bb
diff --git a/README.md b/README.md
index ff0397137..2fb06257f 100644
--- a/README.md
+++ b/README.md
@@ -96,8 +96,8 @@ automl.fit(X_train=X_train, y_train=y_train,
**automl_settings)
# Predict
print(automl.predict_proba(X_train))
-# Export the best model
-print(automl.model)
+# Print the best model
+print(automl.model.estimator)
```
* A basic regression example.
@@ -120,8 +120,8 @@ automl.fit(X_train=X_train, y_train=y_train,
**automl_settings)
# Predict
print(automl.predict(X_train))
-# Export the best model
-print(automl.model)
+# Print the best model
+print(automl.model.estimator)
```
* Time series forecasting.
diff --git a/docs/index.rst b/docs/index.rst
index 8b3f5f3fc..1cad75cd3 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -45,19 +45,6 @@ Online AutoML
:members:
-NLP
----
-
-.. autoclass:: flaml.nlp.AutoTransformers
- :members:
-
-.. autoclass:: flaml.nlp.AzureUtils
- :members:
-
-.. autoclass:: flaml.nlp.JobID
- :members:
-
-
.. Indices and tables
.. ==================
diff --git a/flaml/automl.py b/flaml/automl.py
index e5c5a52db..9bd818d3e 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -1804,7 +1804,7 @@ class AutoML:
better = True # whether we find a better model in one trial
if self._ensemble:
self.best_model = {}
- if self._max_iter < 2 and self.estimator_list:
+ if self._max_iter < 2 and self.estimator_list and self._retrain_final:
# when max_iter is 1, no need to search
# TODO: otherwise, need to make sure SearchStates.init_config is inside search space
self._max_iter = 0
diff --git a/flaml/default/mine.py b/flaml/default/mine.py
deleted file mode 100644
index c4c2b4d82..000000000
--- a/flaml/default/mine.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from typing import List, Tuple, Callable, Optional
-from numpy.typing import ArrayLike
-
-
-class Task:
- name: str
- input: dict # input to automl
- metadata: ArrayLike # featurized metadata
-
-
-def mine_defaults(
- tasks: List[Task],
- regret_bound: float,
- config_predictor: Optional[Callable[[ArrayLike], dict]] = None,
-) -> Tuple[Callable[[ArrayLike], dict], List[float]]:
- '''
- Args:
- tasks: List[Task] | benchmark tasks
- regret_bound: float | maximal regret desired
- config_predictor: None or Callable[[ArrayLike], dict] |
- The existing task -> config predictor
-
- Returns:
- config_predictor: Callable[[ArrayLike], dict] |
- The new task -> config predictor
- regret: List[float] | the regret on each task
- '''
- pass
diff --git a/flaml/nlp/README.md b/flaml/nlp/README.md
deleted file mode 100644
index 3ff6b33b9..000000000
--- a/flaml/nlp/README.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# Hyperparameter Optimization for Huggingface Transformers
-
-AutoTransformers is an AutoML class for fine-tuning pre-trained language models based on the transformers library.
-
-An example of using AutoTransformers:
-
-```python
-from flaml.nlp.autotransformers import AutoTransformers
-
-autohf = AutoTransformers()
-preparedata_setting = {
- "dataset_subdataset_name": "glue:mrpc",
- "pretrained_model_size": "electra-base-discriminator:base",
- "data_root_path": "data/",
- "max_seq_length": 128,
-}
-autohf.prepare_data(**preparedata_setting)
-autohf_settings = {"resources_per_trial": {"gpu": 1, "cpu": 1},
- "num_samples": -1, # unlimited sample size
- "time_budget": 3600,
- "ckpt_per_epoch": 1,
- "fp16": False,
- }
-validation_metric, analysis = autohf.fit(**autohf_settings)
-
-```
-
-The current use cases that are supported:
-
-1. A simplified version of fine-tuning the GLUE dataset using HuggingFace;
-2. For selecting better search space for fine-tuning the GLUE dataset;
-3. Use the search algorithms in flaml for more efficient fine-tuning of HuggingFace.
-
-The use cases that can be supported in future:
-
-1. HPO fine-tuning for text generation;
-2. HPO fine-tuning for question answering.
-
-## Troubleshooting fine-tuning HPO for pre-trained language models
-
-To reproduce the results for our ACL2021 paper:
-
-* [An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models](https://arxiv.org/abs/2106.09204). Xueqing Liu, Chi Wang. ACL-IJCNLP 2021.
-
-```bibtex
-@inproceedings{liu2021hpo,
- title={An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models},
- author={Xueqing Liu and Chi Wang},
- year={2021},
- booktitle={ACL-IJCNLP},
-}
-```
-
-Please refer to the following jupyter notebook: [Troubleshooting HPO for fine-tuning pre-trained language models](https://github.com/microsoft/FLAML/blob/main/notebook/research/acl2021.ipynb)
\ No newline at end of file
diff --git a/flaml/nlp/__init__.py b/flaml/nlp/__init__.py
deleted file mode 100644
index 32bae1503..000000000
--- a/flaml/nlp/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .autotransformers import AutoTransformers
-from .result_analysis.azure_utils import AzureUtils, JobID
diff --git a/flaml/nlp/autotransformers.py b/flaml/nlp/autotransformers.py
deleted file mode 100644
index 096f2f986..000000000
--- a/flaml/nlp/autotransformers.py
+++ /dev/null
@@ -1,887 +0,0 @@
-import json
-import os
-import numpy as np
-import time
-
-try:
- import ray
- import transformers
- from transformers import TrainingArguments
- import datasets
- from .dataset.task_auto import get_default_task
- from .result_analysis.azure_utils import JobID
- from .huggingface.trainer import TrainerForAutoTransformers
-except ImportError:
- print("To use the nlp component in flaml, run pip install flaml[nlp]")
-
-task_list = [
- "seq-classification",
- "regression",
- "question-answering"
-]
-
-
-class AutoTransformers:
- '''The AutoTransformers class
-
- Example:
-
- .. code-block:: python
-
- autohf = AutoTransformers()
- autohf_settings = {
- "resources_per_trial": {"cpu": 1, "gpu": 1},
- "num_samples": -1,
- "time_budget": 60,
- }
-
- validation_metric, analysis = autohf.fit(**autohf_settings)
-
- '''
-
- @staticmethod
- def _convert_dict_to_ray_tune_space(config_json, mode="grid"):
- search_space = {}
-
- if mode == "grid":
- # TODO add test
- for each_hp in config_json.keys():
- this_config = config_json[each_hp]
- assert isinstance(this_config, dict) or isinstance(this_config, list), \
- "config of " + each_hp + " must be dict or list for grid search"
- search_space[each_hp] = ray.tune.grid_search(this_config)
- else:
- for each_hp in config_json.keys():
- this_config = config_json[each_hp]
- assert isinstance(this_config, dict) or isinstance(this_config, list), \
- "config of " + each_hp + " must be dict or list"
- if isinstance(this_config, dict):
- lower = this_config["l"]
- upper = this_config["u"]
- space = this_config["space"]
- if space == "log":
- search_space[each_hp] = ray.tune.loguniform(lower, upper)
- elif space == "linear":
- search_space[each_hp] = ray.tune.uniform(lower, upper)
- elif space == "quniform":
- search_space[each_hp] = ray.tune.quniform(lower, upper, this_config["interval"])
- else:
- search_space[each_hp] = ray.tune.choice(this_config)
-
- return search_space
-
- def _set_search_space(self,
- **custom_hpo_args):
- from .hpo.hpo_searchspace import AutoHPOSearchSpace
-
- search_space_hpo_json \
- = AutoHPOSearchSpace.from_model_and_dataset_name(self.jobid_config.spa,
- self.jobid_config.pre,
- self.jobid_config.presz,
- self.jobid_config.dat,
- self.jobid_config.subdat,
- **custom_hpo_args)
- self._search_space_hpo = AutoTransformers._convert_dict_to_ray_tune_space(
- search_space_hpo_json,
- mode=self.jobid_config.mod)
-
- @staticmethod
- def _get_split_name(data_raw, fold_name=None):
- # TODO coverage
- if fold_name:
- return fold_name
- fold_keys = data_raw.keys()
- if fold_keys == {"train", "validation", "test"}:
- return "train", "validation", "test"
- for each_key in fold_keys:
- for each_split_name in {"train", "validation", "test"}:
- assert not (each_key.startswith(each_split_name) and each_key != each_split_name), \
- "Dataset split must be within {}, must be explicitly specified in dataset_config, e.g.," \
- "'fold_name': ['train','validation_matched','test_matched']. Please refer to the example in the " \
- "documentation of AutoTransformers.prepare_data()".format(",".join(fold_keys))
- return "train", "validation", "test"
-
- def prepare_data(self,
- data_root_path,
- jobid_config=None,
- is_wandb_on=False,
- server_name=None,
- max_seq_length=128,
- fold_name=None,
- resplit_portion=None,
- **custom_data_args):
- """Prepare data
-
- Example:
-
- .. code-block:: python
-
- preparedata_setting = {"server_name": "tmdev", "data_root_path": "data/", "max_seq_length": 128,
- "jobid_config": jobid_config, "wandb_utils": wandb_utils,
- "resplit_portion": {"source": ["train", "validation"],
- "train": [0, 0.8], "validation": [0.8, 0.9], "test": [0.9, 1.0]}}
-
- autohf.prepare_data(**preparedata_setting)
-
- Args:
- server_name:
- A string variable, which can be tmdev or azureml
- data_root_path:
- The root path for storing the checkpoints and output results, e.g., "data/"
- jobid_config:
- A JobID object describing the profile of job
- wandb_utils:
- A WandbUtils object for wandb operations
- max_seq_length (optional):
- Max_seq_lckpt_per_epochength for the huggingface, this hyperparameter must be specified
- at the data processing step
- resplit_portion:
- The proportion for resplitting the train and dev data when split_mode="resplit".
- If args.resplit_mode = "rspt", resplit_portion is required
- is_wandb_on:
- A boolean variable indicating whether wandb is used
- """
- from .dataset.dataprocess_auto import AutoEncodeText
- from transformers import AutoTokenizer
- from datasets import load_dataset
- from .utils import PathUtils
- from .utils import load_dft_args
-
- self._max_seq_length = max_seq_length
- self._server_name = server_name if server_name is not None else "tmdev"
- """
- loading the jobid config from console args
- """
- console_args = load_dft_args()
- self.jobid_config = JobID(console_args)
- if jobid_config:
- self.jobid_config = jobid_config
- if len(custom_data_args) > 0:
- self.jobid_config.set_jobid_from_console_args(console_args=custom_data_args)
- if is_wandb_on:
- from .result_analysis.wandb_utils import WandbUtils
- self.wandb_utils = WandbUtils(is_wandb_on=is_wandb_on,
- wandb_key_path=console_args.key_path,
- jobid_config=self.jobid_config)
- self.wandb_utils.set_wandb_per_run()
- else:
- self.wandb_utils = None
-
- self.path_utils = PathUtils(self.jobid_config, hpo_data_root_path=data_root_path)
-
- if self.jobid_config.spt == "rspt":
- assert resplit_portion, "If split mode is 'rspt', the resplit_portion must be provided. Please " \
- "refer to the example in the documentation of AutoTransformers.prepare_data()"
- if self.jobid_config.subdat:
- data_raw = load_dataset(JobID.dataset_list_to_str(self.jobid_config.dat),
- self.jobid_config.subdat)
- else:
- data_raw = load_dataset(*self.jobid_config.dat)
-
- self._train_name, self._dev_name, self._test_name = AutoTransformers._get_split_name(
- data_raw,
- fold_name=fold_name)
- auto_tokentoids_config = {"max_seq_length": self._max_seq_length}
- self._tokenizer = AutoTokenizer.from_pretrained(self.jobid_config.pre_full, use_fast=True)
-
- def autoencodetext_from_model_and_dataset_name():
- return AutoEncodeText.from_model_and_dataset_name(
- data_raw,
- self.jobid_config.pre_full,
- self.jobid_config.dat,
- self.jobid_config.subdat,
- **auto_tokentoids_config)
-
- data_encoded = autoencodetext_from_model_and_dataset_name()
- self._max_seq_length = 0
- """
- Update the max_seq_length to the minimum of the actual max seq length and the user defined max_seq_length
- """
- for each_fold in data_encoded.keys():
- self._max_seq_length = max(self._max_seq_length,
- max([sum(data_encoded[each_fold][x]['attention_mask']) for x in
- range(len(data_encoded[each_fold]))]))
- self._max_seq_length = int((self._max_seq_length + 15) / 16) * 16
- data_encoded = autoencodetext_from_model_and_dataset_name()
-
- if self.jobid_config.spt == "rspt":
- all_folds_from_source = []
- assert "source" in resplit_portion.keys(), "Must specify the source for resplitting the dataset in" \
- "resplit_portion, which is a list of folder names, e.g., " \
- "resplit_portion = {'source': ['train']}"
-
- source_fold_names = resplit_portion['source']
- for each_fold_name in source_fold_names:
- this_fold_dataset = data_encoded[each_fold_name]
- all_folds_from_source.append(this_fold_dataset)
-
- merged_folds_from_source = datasets.concatenate_datasets(all_folds_from_source)
- merged_folds_from_source = merged_folds_from_source.shuffle(seed=self.jobid_config.sddt)
-
- assert "train" in resplit_portion.keys() and "validation" in resplit_portion.keys() \
- and "test" in resplit_portion.keys(), "train, validation, test must exist in resplit_portion"
-
- for key in ["train", "validation", "test"]:
- target_fold_start, target_fold_end = \
- int(resplit_portion[key][0] * len(merged_folds_from_source)), \
- int(resplit_portion[key][1] * len(merged_folds_from_source))
- subfold_dataset = merged_folds_from_source.select(
- [x for x in range(target_fold_start, target_fold_end)]).flatten_indices()
- if key == "train":
- self.train_dataset = subfold_dataset
- elif key == "validation":
- self.eval_dataset = subfold_dataset
- else:
- self.test_dataset = subfold_dataset
- else:
- self.train_dataset, self.eval_dataset, self.test_dataset \
- = data_encoded[self._train_name], data_encoded[self._dev_name], data_encoded[self._test_name]
-
- def _load_model(self,
- checkpoint_path=None,
- per_model_config=None):
- from transformers import AutoConfig
- from .huggingface.switch_head_auto import AutoSeqClassificationHead, MODEL_CLASSIFICATION_HEAD_MAPPING
-
- this_task = get_default_task(self.jobid_config.dat,
- self.jobid_config.subdat)
- if this_task == "seq-classification":
- self._num_labels = len(self.train_dataset.features["label"].names)
- elif this_task == "regression":
- self._num_labels = 1
-
- if not checkpoint_path:
- checkpoint_path = self.jobid_config.pre_full
-
- def get_this_model():
- from transformers import AutoModelForSequenceClassification
- return AutoModelForSequenceClassification.from_pretrained(checkpoint_path, config=model_config)
-
- def is_pretrained_model_in_classification_head_list():
- return self.jobid_config.pre in MODEL_CLASSIFICATION_HEAD_MAPPING.keys()
-
- def _set_model_config():
- if per_model_config and len(per_model_config) > 0:
- model_config = AutoConfig.from_pretrained(
- checkpoint_path,
- num_labels=model_config_num_labels,
- **per_model_config)
- else:
- model_config = AutoConfig.from_pretrained(
- checkpoint_path,
- num_labels=model_config_num_labels)
- return model_config
-
- if this_task == "seq-classification":
- num_labels_old = AutoConfig.from_pretrained(checkpoint_path).num_labels
- if is_pretrained_model_in_classification_head_list():
- model_config_num_labels = num_labels_old
- else:
- model_config_num_labels = self._num_labels
- model_config = _set_model_config()
-
- if is_pretrained_model_in_classification_head_list():
- # TODO coverage
- if self._num_labels != num_labels_old:
- this_model = get_this_model()
- model_config.num_labels = self._num_labels
- this_model.num_labels = self._num_labels
- this_model.classifier = AutoSeqClassificationHead \
- .from_model_type_and_config(self.jobid_config.pre,
- model_config)
- else:
- this_model = get_this_model()
- else:
- this_model = get_this_model()
-
- this_model.resize_token_embeddings(len(self._tokenizer))
- return this_model
- elif this_task == "regression":
- # TODO add test
- model_config_num_labels = 1
- model_config = _set_model_config()
- this_model = get_this_model()
- return this_model
-
- def _get_metric_func(self):
- data_name = JobID.dataset_list_to_str(self.jobid_config.dat)
- if data_name in ("glue", "super_glue"):
- metric = datasets.load.load_metric(data_name, self.jobid_config.subdat)
- # TODO delete
- elif data_name in ("squad", "squad_v2"):
- metric = datasets.load.load_metric(data_name)
- else:
- metric = datasets.load.load_metric(self.metric_name)
- return metric
-
- def _compute_metrics_by_dataset_name(self,
- eval_pred):
- # TODO coverage
- predictions, labels = eval_pred
- predictions = np.squeeze(predictions) \
- if self.task_name == "regression" else np.argmax(predictions, axis=1)
- metric_func = self._get_metric_func()
- return metric_func.compute(predictions=predictions, references=labels)
-
- def _compute_checkpoint_freq(self,
- num_train_epochs,
- batch_size):
- # TODO coverage
- if "gpu" in self._resources_per_trial:
- ckpt_step_freq = int(min(num_train_epochs, 1) * len(self.train_dataset) / batch_size
- / self._resources_per_trial["gpu"] / self.ckpt_per_epoch) + 1
- else:
- ckpt_step_freq = int(min(num_train_epochs, 1) * len(self.train_dataset) / batch_size
- / self._resources_per_trial["cpu"] / self.ckpt_per_epoch) + 1
-
- return ckpt_step_freq
-
- @staticmethod
- def _separate_config(config):
-
- training_args_config = {}
- per_model_config = {}
-
- for key in config.keys():
- if key in TrainingArguments.__dict__.keys():
- training_args_config[key] = config[key]
- else:
- per_model_config[key] = config[key]
-
- return training_args_config, per_model_config
-
- def _objective(self, config, reporter, checkpoint_dir=None):
- # TODO add test
- from transformers.trainer_utils import set_seed
- self._set_transformers_verbosity(self._transformers_verbose)
-
- def model_init():
- return self._load_model()
- set_seed(config["seed"])
-
- training_args_config, per_model_config = AutoTransformers._separate_config(config)
- this_model = self._load_model(per_model_config=per_model_config)
-
- trial_id = reporter.trial_id
- self.path_utils.make_dir_per_trial(trial_id)
-
- ckpt_freq = self._compute_checkpoint_freq(
- num_train_epochs=config["num_train_epochs"],
- batch_size=config["per_device_train_batch_size"])
-
- assert self.path_utils.ckpt_dir_per_trial
-
- if transformers.__version__.startswith("3"):
- training_args = TrainingArguments(
- output_dir=self.path_utils.ckpt_dir_per_trial,
- do_eval=True,
- per_device_eval_batch_size=32,
- eval_steps=ckpt_freq,
- evaluate_during_training=True,
- save_steps=ckpt_freq,
- save_total_limit=0,
- fp16=self._fp16,
- **training_args_config,
- )
- else:
- from transformers import IntervalStrategy
- training_args = TrainingArguments(
- output_dir=self.path_utils.ckpt_dir_per_trial,
- do_eval=True,
- per_device_eval_batch_size=32,
- eval_steps=ckpt_freq,
- evaluation_strategy=IntervalStrategy.STEPS,
- save_steps=ckpt_freq,
- save_total_limit=0,
- fp16=self._fp16,
- **training_args_config,
- )
-
- trainer = TrainerForAutoTransformers(
- model=this_model,
- args=training_args,
- model_init=model_init,
- train_dataset=self.train_dataset,
- eval_dataset=self.eval_dataset,
- tokenizer=self._tokenizer,
- compute_metrics=self._compute_metrics_by_dataset_name,
- )
- trainer.trial_id = reporter.trial_id
-
- """
- create a wandb run. If os.environ["WANDB_MODE"] == "offline", run = None
- """
-
- if self.wandb_utils:
- run = self.wandb_utils.set_wandb_per_trial()
- import wandb
- for each_hp in config:
- wandb.log({each_hp: config[each_hp]})
- else:
- run = None
-
- trainer.train()
- trainer.evaluate(self.eval_dataset)
- """
- If a wandb run was created, close the run after train and evaluate finish
- """
- if run:
- run.finish()
-
- def _verify_init_config(self,
- **custom_hpo_args):
- for key in custom_hpo_args.keys():
- if key == "points_to_evaluate":
- for each_init_config in custom_hpo_args[key]:
- for each_hp in each_init_config.keys():
- assert each_hp in self._search_space_hpo.keys(), \
- "points_to_evaluate hp must be within the search space"
-
- assert isinstance(each_init_config[each_hp], int) or \
- isinstance(each_init_config[each_hp], float) or \
- isinstance(each_init_config[each_hp], str) or \
- isinstance(each_init_config[each_hp], bool), " points_to_evaluate must be a scalar"
-
- assert isinstance(self._search_space_hpo[each_hp], ray.tune.sample.Categorical) or \
- isinstance(self._search_space_hpo[each_hp], ray.tune.sample.Float) or \
- isinstance(self._search_space_hpo[each_hp], ray.tune.sample.Integer), \
- "Every hp space must either be categorical, integer or float"
-
- if isinstance(self._search_space_hpo[each_hp], ray.tune.sample.Categorical):
- assert each_init_config[each_hp] in self._search_space_hpo[each_hp].categories, \
- "points_to_evaluate {each_hp} value must be within the search space"
- else:
- assert self._search_space_hpo[each_hp].lower <= each_init_config[each_hp] <= \
- self._search_space_hpo[each_hp].upper, \
- "points_to_evaluate {each_hp} value must be within the search space"
-
- def _get_search_algo(self,
- search_algo_name,
- search_algo_args_mode,
- time_budget,
- metric_name,
- metric_mode_name,
- **custom_hpo_args):
- from .hpo.searchalgo_auto import AutoSearchAlgorithm
-
- if search_algo_name in ("bs", "cfo"):
- self._verify_init_config(**custom_hpo_args)
- search_algo = AutoSearchAlgorithm.from_method_name(
- search_algo_name,
- search_algo_args_mode,
- self._search_space_hpo,
- time_budget,
- metric_name,
- metric_mode_name,
- **custom_hpo_args)
- return search_algo
-
- @staticmethod
- def _recover_checkpoint(tune_checkpoint_dir):
- assert tune_checkpoint_dir
- # Get subdirectory used for Huggingface.
- subdirs = [
- os.path.join(tune_checkpoint_dir, name)
- for name in os.listdir(tune_checkpoint_dir)
- if os.path.isdir(os.path.join(tune_checkpoint_dir, name))
- ]
- # There should only be 1 subdir.
- assert len(subdirs) == 1, subdirs
- return subdirs[0]
-
- def _save_ckpt_json(self,
- best_ckpt):
- json.dump({"best_ckpt": best_ckpt},
- open(os.path.join(self.path_utils.result_dir_per_run,
- "save_ckpt_" + self.jobid_config.to_jobid_string() + ".json"), "w"))
-
- def _save_output_metric(self,
- output_metrics):
- json.dump(output_metrics, open(
- os.path.join(self.path_utils.result_dir_per_run,
- "output_metric_" + self.jobid_config.to_jobid_string() + ".json"), "w"))
-
- def _load_ckpt_json(self,
- ckpt_dir=None,
- **kwargs):
- if not ckpt_dir:
- ckpt_dir = os.path.join(self.path_utils.result_dir_per_run,
- "save_ckpt_" + self.jobid_config.to_jobid_string() + ".json")
- try:
- ckpt_json = json.load(open(ckpt_dir))
- return ckpt_json["best_ckpt"]
- except FileNotFoundError as err:
- print("Saved checkpoint not found. Please make sure checkpoint is stored under {}".format(ckpt_dir))
- raise err
-
- def _set_metric(self, custom_metric_name=None, custom_metric_mode_name=None):
- from .dataset.metric_auto import get_default_and_alternative_metric
- from .utils import _variable_override_default_alternative
-
- default_metric, default_mode, all_metrics, all_modes = \
- get_default_and_alternative_metric(
- dataset_name_list=self.jobid_config.dat,
- subdataset_name=self.jobid_config.subdat,
- custom_metric_name=custom_metric_name,
- custom_metric_mode_name=custom_metric_mode_name)
- _variable_override_default_alternative(self,
- "metric_name",
- default_metric,
- all_metrics,
- custom_metric_name)
- _variable_override_default_alternative(self,
- "metric_mode_name",
- default_mode,
- all_modes,
- custom_metric_mode_name)
- self._all_metrics = all_metrics
- self._all_modes = all_modes
-
- def _set_task(self):
- self.task_name = get_default_task(self.jobid_config.dat,
- self.jobid_config.subdat)
-
- def fit_hf(self,
- resources_per_trial,
- num_samples,
- time_budget,
- custom_metric_name=None,
- custom_metric_mode_name=None,
- _fp16=True,
- **custom_hpo_args
- ):
- # TODO remove?
- from transformers.trainer_utils import HPSearchBackend
-
- '''Fine tuning the huggingface using HF's API Transformers.hyperparameter_search (for comparitive purpose).
- Transformers.hyperparameter_search has the following disadvantages:
- (1) it does not return tune.analysis.Analysis result, what is analysis used for
- (2) it is inconvenient to develop on top of Transformers.hyperparameter_search, whose trainable function,
- search space, etc. are defined inside of Transformers.hyperparameter_search.
- An example:
- autohf_settings = {"resources_per_trial": {"cpu": 1},
- "num_samples": 1,
- "time_budget": 100000,
- "ckpt_per_epoch": 1,
- "fp16": False,
- }
- validation_metric, analysis = autohf.fit(**autohf_settings,)
- Args:
- resources_per_trial:
- A dict showing the resources used by each trial,
- e.g., {"gpu": 4, "cpu": 4}
- num_samples:
- An int variable of the maximum number of trials
- time_budget:
- An int variable of the maximum time budget
- custom_metric_name:
- A string of the dataset name or a function,
- e.g., 'accuracy', 'f1', 'loss',
- custom_metric_mode_name:
- A string of the mode name,
- e.g., "max", "min", "last", "all"
- fp16:
- boolean, default = True | whether to use fp16
- custom_hpo_args:
- The additional keyword arguments, e.g.,
- custom_hpo_args = {"points_to_evaluate": [{
- "num_train_epochs": 1,
- "per_device_train_batch_size": 128, }]}
- Returns:
- validation_metric:
- a dict storing the validation score
- '''
-
- def model_init():
- return self._load_model()
-
- def ray_hp_space(trial):
- return {
- "learning_rate": ray.tune.loguniform(1e-6, 1e-4),
- "num_train_epochs": ray.tune.choice(list(range(1, 6))),
- "seed": ray.tune.quniform(1, 41, 1),
- "per_device_train_batch_size": ray.tune.choice([4, 8, 16, 32, 64]),
- }
-
- self._set_metric(custom_metric_name, custom_metric_mode_name)
- self._set_task()
-
- training_args = TrainingArguments(
- output_dir=self.path_utils.hpo_ckpt_path,
- fp16=_fp16,
- )
- this_model = self._load_model()
-
- trainer = TrainerForAutoTransformers(
- this_model,
- training_args,
- model_init=model_init,
- train_dataset=self.train_dataset,
- eval_dataset=self.eval_dataset,
- tokenizer=self._tokenizer,
- compute_metrics=self._compute_metrics_by_dataset_name,
- )
- self.path_utils.make_dir_per_run()
-
- start_time = time.time()
- best_run = trainer.hyperparameter_search(
- n_trials=num_samples,
- time_budget_s=time_budget,
- # hp_space=ray_hp_space,
- backend=HPSearchBackend.RAY,
- resources_per_trial=resources_per_trial)
- duration = time.time() - start_time
- self.last_run_duration = duration
- print("Total running time: {} seconds".format(duration))
-
- hp_dict = best_run.hyperparameters
- hp_dict["seed"] = int(hp_dict["seed"])
-
- best_training_args = TrainingArguments(
- output_dir=self.path_utils.hpo_ckpt_path,
- fp16=_fp16,
- **hp_dict,
- )
-
- best_trainer = TrainerForAutoTransformers(
- this_model,
- best_training_args,
- model_init=model_init,
- train_dataset=self.train_dataset,
- eval_dataset=self.eval_dataset,
- tokenizer=self._tokenizer,
- compute_metrics=self._compute_metrics_by_dataset_name,
- )
-
- best_model_checkpoint_path = os.path.join(self.path_utils.hpo_ckpt_path, "hpo_hf")
- if not os.path.exists(best_model_checkpoint_path):
- os.mkdir(best_model_checkpoint_path)
- best_trainer.train()
- best_trainer.save_model(best_model_checkpoint_path)
- self._save_ckpt_json(best_model_checkpoint_path)
- validation_metric = best_trainer.evaluate()
-
- return validation_metric
-
- def _set_transformers_verbosity(self, transformers_verbose):
- # TODO coverage
- if transformers_verbose == transformers.logging.ERROR:
- transformers.logging.set_verbosity_error()
- elif transformers_verbose == transformers.logging.WARNING:
- transformers.logging.set_verbosity_warning()
- elif transformers_verbose == transformers.logging.INFO:
- transformers.logging.set_verbosity_info()
- elif transformers_verbose == transformers.logging.DEBUG:
- transformers.logging.set_verbosity_debug()
- else:
- raise Exception("transformers_verbose must be set to ERROR, WARNING, INFO or DEBUG")
-
- def fit(self,
- num_samples,
- time_budget,
- custom_metric_name=None,
- custom_metric_mode_name=None,
- ckpt_per_epoch=1,
- fp16=True,
- ray_verbose=1,
- transformers_verbose=10,
- resources_per_trial=None,
- ray_local_mode=False,
- **custom_hpo_args):
- """Fine tuning the huggingface using the hpo setting
-
- Example:
-
- .. code-block:: python
-
- autohf_settings = {"resources_per_trial": {"cpu": 1},
- "num_samples": 1,
- "time_budget": 100000,
- "ckpt_per_epoch": 1,
- "fp16": False,
- }
-
- validation_metric, analysis = autohf.fit(**autohf_settings)
-
- Args:
- resources_per_trial:
- A dict showing the resources used by each trial,
- e.g., {"gpu": 4, "cpu": 4}
- num_samples:
- An int variable of the maximum number of trials
- time_budget:
- An int variable of the maximum time budget
- custom_metric_name:
- A string of the dataset name or a function,
- e.g., 'accuracy', 'f1', 'loss'
- custom_metric_mode_name:
- A string of the mode name,
- e.g., "max", "min", "last", "all"
- ckpt_per_epoch:
- An integer value of number of checkpoints per epoch, default = 1
- ray_verbose:
- An integer, default=1 | verbosit of ray,
- transformers_verbose:
- An integer, default=transformers.logging.INFO | verbosity of transformers, must be chosen from one of
- transformers.logging.ERROR, transformers.logging.INFO, transformers.logging.WARNING,
- or transformers.logging.DEBUG
- fp16:
- A boolean, default = True | whether to use fp16
- ray_local_mode:
- A boolean, default = False | whether to use the local mode (debugging mode) for ray tune.run
- custom_hpo_args:
- The additional keyword arguments, e.g., custom_hpo_args = {"points_to_evaluate": [{
- "num_train_epochs": 1, "per_device_train_batch_size": 128, }]}
-
- Returns:
-
- validation_metric: A dict storing the validation score
-
- analysis: A ray.tune.analysis.Analysis object storing the analysis results from tune.run
- """
- from .hpo.scheduler_auto import AutoScheduler
- self._transformers_verbose = transformers_verbose
-
- """
- Specify the other parse of jobid configs from custom_hpo_args, e.g., if the search algorithm was not specified
- previously, can specify the algorithm here
- """
- if len(custom_hpo_args) > 0:
- self.jobid_config.set_jobid_from_console_args(console_args=custom_hpo_args)
-
- self._resources_per_trial = resources_per_trial
- self._set_metric(custom_metric_name, custom_metric_mode_name)
- self._set_task()
- self._fp16 = fp16
- ray.shutdown()
- ray.init(local_mode=ray_local_mode)
- self._set_search_space(**custom_hpo_args)
-
- search_algo = self._get_search_algo(self.jobid_config.alg,
- self.jobid_config.arg,
- time_budget,
- self.metric_name,
- self.metric_mode_name,
- **custom_hpo_args)
- scheduler = AutoScheduler.from_scheduler_name(self.jobid_config.pru)
- self.ckpt_per_epoch = ckpt_per_epoch
- self.path_utils.make_dir_per_run()
-
- assert self.path_utils.ckpt_dir_per_run
- start_time = time.time()
-
- tune_config = self._search_space_hpo
- tune_config["seed"] = self.jobid_config.sdhf
-
- analysis = ray.tune.run(
- self._objective,
- metric=self.metric_name,
- mode=self.metric_mode_name,
- name="ray_result",
- resources_per_trial=resources_per_trial,
- config=tune_config,
- verbose=ray_verbose,
- local_dir=self.path_utils.ckpt_dir_per_run,
- num_samples=num_samples,
- time_budget_s=time_budget,
- keep_checkpoints_num=1,
- scheduler=scheduler,
- search_alg=search_algo,
- )
- duration = time.time() - start_time
- self.last_run_duration = duration
- print("Total running time: {} seconds".format(duration))
-
- ray.shutdown()
-
- best_trial = analysis.get_best_trial(scope="all", metric=self.metric_name, mode=self.metric_mode_name)
- validation_metric = {"eval_" + self.metric_name
- : best_trial.metric_analysis[self.metric_name][self.metric_mode_name]}
- for x in range(len(self._all_metrics)):
- validation_metric["eval_" + self._all_metrics[x]] \
- = best_trial.metric_analysis[self._all_metrics[x]][self._all_modes[x]]
-
- get_best_ckpt = analysis.get_best_checkpoint(best_trial, metric=self.metric_name, mode=self.metric_mode_name)
- best_ckpt = AutoTransformers._recover_checkpoint(get_best_ckpt)
-
- self._save_ckpt_json(best_ckpt)
-
- return validation_metric, analysis
-
- def predict(self,
- ckpt_json_dir=None,
- **kwargs):
- '''Predict label for test data.
-
- An example:
- predictions, test_metric = autohf.predict()
-
- Args:
- ckpt_json_dir:
- the checkpoint for the fine-tuned huggingface if you wish to override
- the saved checkpoint in the training stage under self.path_utils._result_dir_per_run
-
- Returns:
- A numpy array of shape n * 1 - - each element is a predicted class
- label for an instance.
- '''
- best_checkpoint = self._load_ckpt_json(ckpt_json_dir, **kwargs)
- best_model = self._load_model(checkpoint_path=best_checkpoint)
- training_args = TrainingArguments(per_device_eval_batch_size=1,
- output_dir=self.path_utils.result_dir_per_run)
- test_trainer = TrainerForAutoTransformers(best_model, training_args)
-
- if self.jobid_config.spt == "ori":
- # TODO add test
- if "label" in self.test_dataset.features.keys():
- self.test_dataset.remove_columns_("label")
- print("Cleaning the existing label column from test data")
-
- test_dataloader = test_trainer.get_test_dataloader(self.test_dataset)
- predictions, labels, _ = test_trainer.prediction_loop(test_dataloader, description="Prediction")
- predictions = np.squeeze(predictions) \
- if get_default_task(self.jobid_config.dat,
- self.jobid_config.subdat) == "regression" \
- else np.argmax(predictions, axis=1)
-
- if self.jobid_config.spt == "rspt":
- assert labels is not None
- metric = self._get_metric_func()
- output_metric = metric.compute(predictions=predictions, references=labels)
- self._save_output_metric(output_metric)
- return predictions, output_metric
- else:
- return predictions, None
-
- def output_prediction(self,
- predictions=None,
- output_prediction_path=None,
- output_zip_file_name=None):
- """
- When using the original GLUE split, output the prediction on test data,
- and prepare the .zip file for submission
-
- Example:
- local_archive_path = self.autohf.output_prediction(predictions,
- output_prediction_path= self.console_args.data_root_dir + "result/",
- output_zip_file_name=azure_save_file_name)
-
- Args:
- predictions:
- A list of predictions, which is the output of AutoTransformers.predict()
- output_prediction_path:
- Output path for the prediction
- output_zip_file_name:
- An string, which is the name of the output zip file
-
- Returns:
- The path of the output .zip file
- """
- from .dataset.submission_auto import auto_output_prediction
- return auto_output_prediction(self.jobid_config.dat,
- output_prediction_path,
- output_zip_file_name,
- predictions,
- self.train_dataset,
- self._dev_name,
- self.jobid_config.subdat)
diff --git a/flaml/nlp/dataset/__init__.py b/flaml/nlp/dataset/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/flaml/nlp/dataset/dataprocess_auto.py b/flaml/nlp/dataset/dataprocess_auto.py
deleted file mode 100644
index 09e421891..000000000
--- a/flaml/nlp/dataset/dataprocess_auto.py
+++ /dev/null
@@ -1,230 +0,0 @@
-from collections import OrderedDict
-from functools import partial
-
-from transformers import AutoTokenizer
-from .sentence_keys_auto import get_sentence_keys
-
-
-def inserting_sepp(sent, start, end, this_tokenizer):
- return \
- sent[:start].rstrip() + " " + this_tokenizer.sep_token + " " + sent[start:end] \
- + " " + this_tokenizer.sep_token + " " + sent[end:].lstrip()
-
-
-def tokenize_superglue_copa(this_example,
- this_tokenizer,
- dataset_name,
- subdataset_name=None,
- **kwargs):
- return None
-
-
-def tokenize_superglue_wic_gpt2(this_example,
- this_tokenizer,
- dataset_name,
- subdataset_name=None,
- **kwargs):
- return None
-
-
-def tokenize_superglue_wic(this_example,
- this_tokenizer,
- dataset_name,
- subdataset_name=None,
- **kwargs
- ):
- """
- tokenize the data from the wic task (word-in-context dataset),
- e.g., sentence 1: "There's a lot of trash on the bed of the river"
- sentence 2: "I keep a glass of water next to my bed when I sleep",
- label = False (different word senses)
- In the superglue data, the position of the word in sentence 1 and 2 are provided
- What this function does is to update the span position after tokenization, based on each LM's own tokenizer,
- The key is to insert an [SEP] before and after the original sentence, then feed it into the LM's tokenizer.
- There are two challenges:
- (1) Each LM's tokenizations are different, e.g., in XLNet's tokenizer, the paddings are on the left'
- (2) Some LM's tokenization would add an underline symbol before the word, e.g., "There's a lot"
- -> [_There, _', _s, _a, _lot]
- When underline meets special char such as '"', "'", the tokenized sequence after adding [SEP] needs to be
- aligned with the sequence tokenized without [SEP]. We use a two pointer algorithm for the alignment
- """
- sent1, sent2 = this_example["sentence1"], this_example["sentence2"]
- start1, end1 = this_example["start1"], this_example["end1"]
- start2, end2 = this_example["start2"], this_example["end2"]
- """
- Add [SEP] to the sentence
- """
- altered_sent1 = inserting_sepp(sent1, start1, end1, this_tokenizer)
- altered_sent2 = inserting_sepp(sent2, start2, end2, this_tokenizer)
- input_ids_sepp = this_tokenizer(*(altered_sent1, altered_sent2),
- padding="max_length",
- max_length=1024,
- truncation=True)["input_ids"]
- data_pair = (sent1, sent2)
- assert "max_seq_length" in kwargs, "max_seq_length must be provided for glue"
- this_data = this_tokenizer(*data_pair, padding="max_length", max_length=kwargs["max_seq_length"], truncation=True)
- input_ids = this_data["input_ids"]
- which_sepp = 0
-
- """
- span_start_end: a 2x2 array:
- * (span_start_end[0][0], span_start_end[0][1]) are the spans of the position of the word in the first sentence
- * (span_start_end[1][0], span_start_end[1][1]) are the spans of the position of the word in the second sentence
- """
- span_start_end = [[-1, -1], [-1, -1]]
-
- ptr_sepp = ptr_nosepp = 0
- try:
- padding_direction = this_tokenizer.padding_side
- if padding_direction == "left":
- # TODO coverage
- padding_id = input_ids_sepp[0]
- while input_ids_sepp[ptr_sepp] == padding_id:
- ptr_sepp += 1
- while input_ids[ptr_nosepp] == padding_id:
- ptr_nosepp += 1
- except KeyError:
- pass
- sep_id = this_tokenizer.convert_tokens_to_ids([this_tokenizer.sep_token])[0]
- """
- use two pointers to align the tokenized sequence before and after adding [SEP];
- ptr_sepp: the pointer after adding; ptr_nosepp: the pointer without adding
- """
- while ptr_sepp < len(input_ids_sepp) and ptr_nosepp < len(input_ids) and \
- input_ids_sepp[ptr_sepp] != 0 and input_ids[ptr_nosepp] != 0:
- if input_ids_sepp[ptr_sepp] == input_ids[ptr_nosepp]:
- ptr_sepp += 1
- ptr_nosepp += 1
- else:
- if not (input_ids_sepp[ptr_sepp] == sep_id
- or this_tokenizer.convert_ids_to_tokens([input_ids_sepp[ptr_sepp]])[0] in ('▁', '_')):
- break
- if input_ids_sepp[ptr_sepp] == sep_id:
- span_start_end[int(which_sepp / 2)][which_sepp % 2] = ptr_nosepp
- which_sepp += 1
- ptr_sepp += 1
- else:
- # TODO coverage
- ptr_sepp += 1
- """
- max_word_span is the maximum tokens of the word
- It is set to 16 following deberta:
- https://github.com/microsoft/DeBERTa/blob/master/DeBERTa/apps/tasks/superglue_tasks.py#L1054
- """
- max_word_span = 16
- word_indices = []
- for idx1 in range(2):
- if span_start_end[idx1][1] < kwargs["max_seq_length"]:
- first_span = [x for x in range(span_start_end[idx1][0], span_start_end[idx1][1])
- if x < kwargs["max_seq_length"]] + [0] * (max_word_span - span_start_end[idx1][1]
- + span_start_end[idx1][0])
- word_indices.append(first_span)
- this_data["word_spans"] = word_indices
- return this_data
-
-
-def tokenize_glue(this_example,
- this_tokenizer,
- dataset_name,
- subdataset_name=None,
- **kwargs):
- sentence_keys = get_sentence_keys(dataset_name, subdataset_name)
-
- if len(sentence_keys) > 1:
- sentence1_key, sentence2_key = sentence_keys[0], sentence_keys[1]
- else:
- # TODO coverage
- sentence1_key = sentence_keys[0]
- sentence2_key = None
-
- data_pair = (
- (this_example[sentence1_key],) if sentence2_key is None else (
- this_example[sentence1_key], this_example[sentence2_key])
- )
- assert "max_seq_length" in kwargs, "max_seq_length must be provided for glue"
- return this_tokenizer(*data_pair, padding="max_length", max_length=kwargs["max_seq_length"], truncation=True)
-
-
-TOKENIZER_MAPPING = OrderedDict(
- [
- (("glue", "rte"), tokenize_glue),
- (("glue", "mrpc"), tokenize_glue),
- (("glue", "cola"), tokenize_glue),
- (("glue", "wnli"), tokenize_glue),
- (("glue", "stsb"), tokenize_glue),
- (("glue", "sst2"), tokenize_glue),
- (("glue", "mnli"), tokenize_glue),
- (("glue", "qqp"), tokenize_glue),
- (("glue", "qnli"), tokenize_glue),
- (("super_glue", "wic"), tokenize_superglue_wic),
- ]
-)
-
-
-class AutoEncodeText:
- """
- This is a generic input text tokenization class that will be instantiated as one of the
- tokenization classes of the library when created with the
- `~flaml.nlp.dataset.AutoEncodeText.from_model_and_dataset_name` class method.
-
- This class cannot be instantiated directly using ``__init__()`` (throws an error).
- """
-
- def __init__(self):
- raise EnvironmentError(
- "AutoEncodeText is designed to be instantiated "
- "using the `AutoEncodeText.from_model_and_dataset_name(cls,"
- "data_raw,model_checkpoint_path,dataset_name,subdataset_name = None,**kwargs)` methods."
- )
-
- @classmethod
- def from_model_and_dataset_name(cls,
- data_raw,
- model_checkpoint_path,
- dataset_name_list: list = None,
- subdataset_name=None,
- **kwargs):
- """
- Instantiate one of the input text tokenization classes from the raw data, model checkpoint path, dataset name
- and sub dataset name. The raw data is used for creating a mapping function from the raw tokens to the
- tokenized token ids.
-
- Args:
- data_raw:
- The raw data (a datasets.Dataset object)
-
- model_checkpoint_path:
- A string variable which specifies the model path, e.g., "google/electra-base-discriminator"
-
- dataset_name_list:
- A list which is the dataset name, e.g., ["glue"]
-
- subdataset_name:
- A string variable which is the sub dataset name,e.g., "rte"
-
- kwargs:
- The values in kwargs of any keys will be used for the mapping function
-
- Examples:
- >>> from datasets import load_dataset
- >>> data_raw = load_dataset("glue", "rte")
- >>> AutoEncodeText.from_model_and_dataset_name(data_raw, "google/electra-base-discriminator", ["glue"], "rte")
-
- """
- from ..result_analysis.azure_utils import JobID
- dataset_name = JobID.dataset_list_to_str(dataset_name_list)
- if (dataset_name, subdataset_name) in TOKENIZER_MAPPING.keys():
- this_tokenizer = AutoTokenizer.from_pretrained(model_checkpoint_path, use_fast=True)
- token_func = TOKENIZER_MAPPING[(dataset_name, subdataset_name)]
- return data_raw.map(
- partial(token_func,
- this_tokenizer=this_tokenizer,
- dataset_name=dataset_name,
- subdataset_name=subdataset_name,
- **kwargs), batched=False)
- raise ValueError(
- "Unrecognized method {},{} for this kind of AutoGridSearchSpace: {}.\n"
- "Method name should be one of {}.".format(
- dataset_name, subdataset_name, cls.__name__, ", ".join(c[0] for c in TOKENIZER_MAPPING.keys())
- )
- )
diff --git a/flaml/nlp/dataset/metric_auto.py b/flaml/nlp/dataset/metric_auto.py
deleted file mode 100644
index 4abda815f..000000000
--- a/flaml/nlp/dataset/metric_auto.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# https://github.com/huggingface/datasets/blob/master/metrics/glue/glue.py
-from collections import OrderedDict
-import typing
-
-metric_mode_mapping_glue = {
- "cola": [("matthews_correlation", "max")],
- "mnli": [("accuracy", "max")],
- "mrpc": [("accuracy", "max"), ("f1", "max")],
- "qnli": [("accuracy", "max")],
- "qqp": [("accuracy", "max"), ("f1", "max")],
- "rte": [("accuracy", "max")],
- "sst2": [("accuracy", "max")],
- "stsb": [("pearson", "max"), ("spearmanr", "max")],
- "wnli": [("accuracy", "max")]
-}
-
-metric_mode_mapping_squad = [("exact_match", "max"), ("f1", "max")]
-
-metric_mode_mapping_super_glue = {
- "axb": [("matthews_correlation", "max")],
- "cb": [("accuracy", "max"), ("f1", "max")],
- "copa": [("accuracy", "max")],
- "rte": [("accuracy", "max")],
- "wic": [("accuracy", "max")],
- "wsc": [("accuracy", "max")],
- "wsc.fixed": [("accuracy", "max")],
- "boolq": [("accuracy", "max")],
- "axg": [("accuracy", "max")]
-}
-
-metric_mode_mapping_imdb = [("accuracy", "max")]
-
-metric_mode_mapping_yelp = [("accuracy", "max")]
-
-METRIC_MAPPING = OrderedDict(
- [
- ("squad", metric_mode_mapping_squad),
- ("glue", metric_mode_mapping_glue),
- ("super_glue", metric_mode_mapping_super_glue),
- ("imdb", metric_mode_mapping_imdb),
- ("yelp_review_full", metric_mode_mapping_yelp)
- ]
-)
-
-
-def get_default_and_alternative_metric(dataset_name_list: typing.List,
- subdataset_name=None,
- custom_metric_name=None,
- custom_metric_mode_name=None):
- from ..result_analysis.azure_utils import JobID
- dataset_name = JobID.dataset_list_to_str(dataset_name_list)
- if dataset_name not in METRIC_MAPPING.keys():
- assert custom_metric_name and custom_metric_mode_name, \
- "The dataset is not in {}, you must explicitly specify " \
- "the custom_metric_name and custom_metric_mode_name".format(",".join(METRIC_MAPPING.keys()))
- eval_name_mapping = METRIC_MAPPING[dataset_name]
- if isinstance(eval_name_mapping, dict):
- assert subdataset_name and subdataset_name in eval_name_mapping, \
- "dataset_name and subdataset_name not correctly specified"
- default_metric, default_mode = eval_name_mapping[subdataset_name][0]
- all_metrics, all_mode \
- = [x[0] for x in eval_name_mapping[subdataset_name]] \
- + ["loss"], [x[1] for x in eval_name_mapping[subdataset_name]] + ["min"]
-
- return default_metric, default_mode, all_metrics, all_mode
- else:
- # TODO coverage
- assert isinstance(eval_name_mapping, list), "dataset_name and subdataset_name not correctly specified"
-
- default_metric, default_mode = eval_name_mapping[0]
- all_metrics, all_mode = [x[0] for x in eval_name_mapping] + ["loss"], \
- [x[1] for x in eval_name_mapping] + ["min"]
-
- return default_metric, default_mode, all_metrics, all_mode
diff --git a/flaml/nlp/dataset/sentence_keys_auto.py b/flaml/nlp/dataset/sentence_keys_auto.py
deleted file mode 100644
index bcf098cf4..000000000
--- a/flaml/nlp/dataset/sentence_keys_auto.py
+++ /dev/null
@@ -1,28 +0,0 @@
-sentence_keys_glue = {
- "cola": ["sentence"],
- "mnli": ["premise", "hypothesis"],
- "mrpc": ["sentence1", "sentence2"],
- "qnli": ["sentence", "question"],
- "qqp": ["question1", "question2"],
- "rte": ["sentence1", "sentence2"],
- "sst2": ["sentence"],
- "stsb": ["sentence1", "sentence2"],
- "wnli": ["sentence1", "sentence2"]
-}
-
-sentence_keys_super_glue = {
- "rte": ["hypothesis", "premise"],
- "wic": ["sentence1", "sentence2"],
- "wsc": ["text"]
-}
-
-
-def get_sentence_keys(dataset_name, subdataset_name=None):
- eval_name_mapping = globals()["sentence_keys_" + dataset_name]
- if isinstance(eval_name_mapping, dict):
- assert subdataset_name and subdataset_name in eval_name_mapping, \
- "dataset_name and subdataset_name not correctly specified"
- sentence_keys = eval_name_mapping[subdataset_name]
- else:
- sentence_keys = eval_name_mapping
- return sentence_keys
diff --git a/flaml/nlp/dataset/submission_auto.py b/flaml/nlp/dataset/submission_auto.py
deleted file mode 100644
index bf49fd3d0..000000000
--- a/flaml/nlp/dataset/submission_auto.py
+++ /dev/null
@@ -1,133 +0,0 @@
-import os
-import shutil
-from collections import OrderedDict
-
-file_name_mapping_glue = {
- "ax": ["AX.tsv"],
- "cola": ["CoLA.tsv"],
- "mnli": ["MNLI-m.tsv", "MNLI-mm.tsv"],
- "mrpc": ["MRPC.tsv"],
- "qnli": ["QNLI.tsv"],
- "qqp": ["QQP.tsv"],
- "rte": ["RTE.tsv"],
- "sst2": ["SST-2.tsv"],
- "stsb": ["STS-B.tsv"],
- "wnli": ["WNLI.tsv"]
-}
-
-default_prediction_glue = {
- "ax": ["entailment"],
- "cola": ["0"],
- "mnli": ["neutral", "neutral"],
- "mrpc": ["0"],
- "qnli": ["not_entailment"],
- "qqp": ["0"],
- "rte": ["not_entailment"],
- "sst2": ["0"],
- "stsb": ["0.0"],
- "wnli": ["0"]
-}
-
-test_size_glue = {
- "ax": [1104],
- "cola": [1064],
- "mnli": [9796, 9847],
- "mrpc": [1725],
- "qnli": [5463],
- "qqp": [390965],
- "rte": [3000],
- "sst2": [1821],
- "stsb": [1379],
- "wnli": [146]
-}
-
-
-def output_prediction_glue(output_path, zip_file_name, predictions, train_data, dev_name, subdataset_name):
- output_dir = os.path.join(output_path, zip_file_name)
- if os.path.exists(output_dir):
- assert os.path.isdir(output_dir)
- else:
- import pathlib
- pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
- if subdataset_name != "stsb":
- label_list = train_data.features["label"].names
-
- output_blank_tsv(output_dir)
- for each_subdataset_name in file_name_mapping_glue.keys():
- for idx in range(len(file_name_mapping_glue[each_subdataset_name])):
- each_file = file_name_mapping_glue[each_subdataset_name][idx]
- if subdataset_name != "mnli":
- is_match = subdataset_name == each_subdataset_name
- else:
- # TODO coverage
- if dev_name == "validation_matched":
- is_match = each_file == "MNLI-m.tsv"
- else:
- is_match = each_file == "MNLI-mm.tsv"
- if is_match:
- with open(os.path.join(output_dir, each_file), "w") as writer:
- writer.write("index\tprediction\n")
- for index, item in enumerate(predictions):
- if subdataset_name == "stsb":
- # TODO coverage
- if item > 5.0:
- item = 5.0
- writer.write(f"{index}\t{item:3.3f}\n")
- else:
- if subdataset_name in ("rte", "qnli", "mnli"):
- # TODO coverage
- item = label_list[item]
- writer.write(f"{index}\t{item}\n")
- else:
- if int(item) == item:
- item = int(item)
- writer.write(f"{index}\t{item}\n")
- else:
- # TODO coverage
- writer.write(f"{index}\t{item:3.3f}\n")
-
- shutil.make_archive(os.path.join(output_path, zip_file_name), 'zip', output_dir)
- return os.path.join(output_path, zip_file_name + ".zip")
-
-
-OUTPUT_PREDICTION_MAPPING = OrderedDict(
- [
- ("glue", output_prediction_glue),
- ]
-)
-
-
-def auto_output_prediction(dataset_name_list: list,
- output_path,
- zip_file_name,
- predictions,
- train_data,
- dev_name,
- subset_name):
- from ..result_analysis.azure_utils import JobID
- dataset_name = JobID.dataset_list_to_str(dataset_name_list)
- if dataset_name in OUTPUT_PREDICTION_MAPPING.keys():
- return OUTPUT_PREDICTION_MAPPING[dataset_name](output_path,
- zip_file_name,
- predictions,
- train_data,
- dev_name,
- subset_name)
- else:
- raise ValueError(
- "Unrecognized dataset {}. \n"
- "Should be one of {}.".format(dataset_name, ", ".join(c.__name__ for c in OUTPUT_PREDICTION_MAPPING.keys())
- )
- )
-
-
-def output_blank_tsv(output_dir):
- for each_subdataset_name in file_name_mapping_glue.keys():
- for idx in range(len(file_name_mapping_glue[each_subdataset_name])):
- each_file = file_name_mapping_glue[each_subdataset_name][idx]
- default_prediction = default_prediction_glue[each_subdataset_name][idx]
- test_size = test_size_glue[each_subdataset_name][idx]
- with open(os.path.join(output_dir, each_file), "w") as writer:
- writer.write("index\tprediction\n")
- for index in range(test_size):
- writer.write(f"{index}\t{default_prediction}\n")
diff --git a/flaml/nlp/dataset/task_auto.py b/flaml/nlp/dataset/task_auto.py
deleted file mode 100644
index 4cdfa03c4..000000000
--- a/flaml/nlp/dataset/task_auto.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# https://github.com/huggingface/datasets/blob/master/metrics/glue/glue.py
-
-from collections import OrderedDict
-
-task_mapping_glue = {
- "cola": "seq-classification",
- "mnli": "seq-classification",
- "mrpc": "seq-classification",
- "qnli": "seq-classification",
- "qqp": "seq-classification",
- "rte": "seq-classification",
- "sst2": "seq-classification",
- "stsb": "regression",
- "wnli": "seq-classification"
-}
-
-task_mapping_squad = "question-answering"
-
-task_mapping_super_glue = {
- "wic": "seq-classification",
- "rte": "seq-classification"
-}
-
-TASK_MAPPING = OrderedDict(
- [
- ("squad", task_mapping_squad),
- ("glue", task_mapping_glue),
- ("super_glue", task_mapping_super_glue),
- ]
-)
-
-
-def get_default_task(dataset_name_list: list, subdataset_name=None):
- from ..result_analysis.azure_utils import JobID
- dataset_name = JobID.dataset_list_to_str(dataset_name_list)
- assert dataset_name in TASK_MAPPING.keys(), "The dataset is not in {}, you must explicitly specify " \
- "the custom_metric_name and custom_metric_mode_name".format(
- ",".join(TASK_MAPPING.keys()))
- eval_name_mapping = TASK_MAPPING[dataset_name]
- if isinstance(eval_name_mapping, dict):
- assert subdataset_name and subdataset_name in eval_name_mapping, \
- "dataset_name and subdataset_name not correctly specified"
- default_task = eval_name_mapping[subdataset_name]
- else:
- # TODO coverage
- assert isinstance(eval_name_mapping, list), "dataset_name and subdataset_name not correctly specified"
- default_task = eval_name_mapping
- return default_task
diff --git a/flaml/nlp/hpo/__init__.py b/flaml/nlp/hpo/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/flaml/nlp/hpo/get_grid_search_space.py b/flaml/nlp/hpo/get_grid_search_space.py
deleted file mode 100644
index a3e59a2b6..000000000
--- a/flaml/nlp/hpo/get_grid_search_space.py
+++ /dev/null
@@ -1,465 +0,0 @@
-# lookup table for the grid configs in each pre-trained language huggingface for different tasks
-
-
-def get_space_union_and_unique(search_space_common,
- search_space_unique,
- this_case_tags: list):
- """
- get the recommended search configs for each pre-trained language models
-
- Args:
- search_space_common:
- the union of configs recommended by the LM for all cases;
- search_space_unique:
- the recommended config by the LM for a specific condition, e.g., small model
- this_case_tags:
- a list, which contains the tags describing the specific condition, e.g., ["small"]
- """
- search_space_union = search_space_common.copy()
- this_search_space = search_space_common.copy()
- # enumerate over each case where the search space is different
- # this difference can be the dataset or model size, etc.
- is_included = False
- from ..utils import merge_dicts
- for each_case in search_space_unique.keys():
- from ..utils import _check_dict_keys_overlaps
- if each_case in this_case_tags:
- is_included = True
- assert not _check_dict_keys_overlaps(this_search_space, search_space_unique[each_case]), \
- "the hyperparameters of common and unique search spaces should not have overlaps"
- this_search_space.update(search_space_unique[each_case])
- search_space_union = merge_dicts(search_space_union, search_space_unique[each_case])
- if is_included:
- return this_search_space
- else:
- if "other" in search_space_unique.keys():
- search_space_union = merge_dicts(search_space_union, search_space_unique["other"])
- return search_space_union
-
-
-def get_deberta_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- DEBERTA: DECODING-ENHANCED BERT WITH DISENTANGLED ATTENTION: Table 9
- https://arxiv.org/abs/2006.03654
- """
- search_space_common = {
- "cls_dropout": [0, 0.1, 0.15],
- "warmup_steps": [50, 100, 500, 1000],
- "per_device_train_batch_size": [16, 32, 48, 64],
- "num_train_epochs": [10],
- "adam_epsilon": [1e-6],
- }
- search_space_unique = {
- "large": {
- "learning_rate": [5e-6, 8e-6, 9e-6, 1e-5],
- "weight_decay": [0.01],
- },
- "base": {
- "learning_rate": [1.5e-5, 2e-5, 3e-5, 4e-5],
- }
- }
- return get_space_union_and_unique(search_space_common, search_space_unique, [model_size_type])
-
-
-def get_longformer_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- TODO: Longformer: The Long-Document Transformer
- """
- return
-
-
-def get_funnel_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing
- https://arxiv.org/abs/2006.03236
- """
- search_space_common = {"learning_rate": [1e-5, 2e-5, 3e-5],
- "hidden_dropout": [0.1],
- "activation_dropout": [0.0],
- "attention_dropout": [0.1],
- "weight_decay": [0.01],
- "warmup_ratio": [0.1],
- "adam_epsilon": [1e-6],
- }
- search_space_unique = {
- "imdb": {
- "per_device_train_batch_size": [32],
- "num_train_epochs": [5]
- },
- "ag_news": {
- "per_device_train_batch_size": [32],
- "num_train_epochs": [3]
- },
- "dbpedia_14": {
- "per_device_train_batch_size": [64],
- "num_train_epochs": [3]
- },
- "yelp_polarity": {
- "per_device_train_batch_size": [128],
- "num_train_epochs": [3]
- },
- "yelp_review_full": {
- "per_device_train_batch_size": [128],
- "num_train_epochs": [3]
- },
- "amazon_polarity": {
- "per_device_train_batch_size": [128],
- "num_train_epochs": [3]
- },
- "amazon_review_multi": {
- "per_device_train_batch_size": [128],
- "num_train_epochs": [3]
- },
- "glue_rte": {
- "per_device_train_batch_size": [16],
- "num_train_epochs": [10]
- },
- "glue_mrpc": {
- "per_device_train_batch_size": [16],
- "num_train_epochs": [10]
- },
- "glue_stsb": {
- "per_device_train_batch_size": [16],
- "num_train_epochs": [10]
- },
- "glue_cola": {
- "per_device_train_batch_size": [16],
- "num_train_epochs": [10]
- },
- "glue_sst2": {
- "per_device_train_batch_size": [32],
- "num_train_epochs": [5]
- },
- "glue_qnli": {
- "per_device_train_batch_size": [32],
- "num_train_epochs": [3]
- },
- "glue_mnli": {
- "per_device_train_batch_size": [64],
- "num_train_epochs": [3]
- },
- "glue_qqp": {
- "per_device_train_batch_size": [64],
- "num_train_epochs": [5]
- }
- }
- from ..result_analysis.azure_utils import JobID
- return get_space_union_and_unique(search_space_common, search_space_unique,
- [JobID.get_full_data_name(
- dataset_name_list,
- subdataset_name)])
-
-
-def get_bert_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding
- https://arxiv.org/pdf/1810.04805.pdf
- """
- search_space_common = {}
- search_space_unique = {
- # Section 4.1: We use a batch size of 32 and fine-tune for 3 epochs over the data for all GLUE tasks. For each
- # task, we selected the best fine-tuning learning rate (among 5e-5, 4e-5, 3e-5, and 2e-5) on the Dev set
- "glue": {
- "learning_rate": [5e-5, 4e-5, 3e-5, 2e-5],
- "per_device_train_batch_size": [32],
- "num_train_epochs": [3],
- },
- # Section 4.2: We fine-tune for 3 epochs with a learning rate of 5e-5 and a batch size of 32
- "squad": {
- "learning_rate": [5e-5],
- "per_device_train_batch_size": [32],
- "num_train_epochs": [2],
- },
- # Section 4.3: We fine-tuned for 2 epochs with a learning rate of 5e-5 and a batch size of 48.
- "squad_v2": {
- "learning_rate": [5e-5],
- "per_device_train_batch_size": [48],
- "num_train_epochs": [2],
- },
- # Section 4.4: We fine-tune the huggingface for 3 epochs with a learning rate of 2e-5 and a batch size of 16.
- "swag": {
- "learning_rate": [2e-5],
- "per_device_train_batch_size": [16],
- "num_train_epochs": [3],
- },
- # Appedix A. The optimal hyperparameter values are task-specific, but we found the following
- # range of possible values to work well across all tasks:
- # - Batch size: 16, 32
- # - Learning rate (Adam): 5e-5, 3e-5, 2e-5
- # - Number of epochs: 2, 3, 4
- "other": {
- "learning_rate": [5e-5, 3e-5, 2e-5],
- "per_device_train_batch_size": [16, 32],
- "num_train_epochs": [2, 3, 4],
- }
- }
- return get_space_union_and_unique(search_space_common,
- search_space_unique,
- dataset_name_list)
-
-
-def get_roberta_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- # RoBERTa: A Robustly Optimized BERT Pretraining Approach
- # https://arxiv.org/pdf/1907.11692.pdf
- search_space_common = {
- "warmup_ratio": [0.06],
- }
- search_space_unique = {
- # Table 10: Hyperparameters for finetuning RoBERTa-LARGE on RACE, SQuAD and GLUE.
- # We consider a limited hyperparameter
- # sweep for each task, with batch sizes ∈ {16, 32}
- # and learning rates ∈ {1e−5, 2e−5, 3e−5}, with a
- # linear warmup for the first 6% of steps followed by
- # a linear decay to 0.
- "glue": {
- "learning_rate": [1e-5, 2e-5, 3e-5],
- "per_device_train_batch_size": [16, 32],
- "weight_decay": [0.1],
- "num_train_epochs": [10],
- },
- "race": {
- "learning_rate": [1e-5],
- "per_device_train_batch_size": [16],
- "weight_decay": [0.1],
- "num_train_epochs": [4],
- },
- "squad": {
- "learning_rate": [1.5e-5],
- "per_device_train_batch_size": [48],
- "weight_decay": [0.01],
- "num_train_epochs": [2],
- }
- }
- return get_space_union_and_unique(search_space_common,
- search_space_unique,
- dataset_name_list)
-
-
-def get_electra_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- ELECTRA: PRE-TRAINING TEXT ENCODERS AS DISCRIMINATORS RATHER THAN GENERATORS
- https://arxiv.org/pdf/2003.10555.pdf
- """
- assert model_size_type in ("small", "base", "large", "intermediate", "xlarge"), \
- "Electra paper has only provided hyperparameter for the small and base huggingface"
- search_space_common = {
- "learning_rate": [3e-5, 5e-5, 1e-4, 1.5e-4],
- "weight_decay": [0.0],
- "adam_epsilon": [1e-6],
- "warmup_ratio": [0.1],
- "per_device_train_batch_size": [32],
- "hidden_dropout_prob": [0.1],
- "attention_probs_dropout_prob": [0.1],
- }
- search_space_unique = {
- # Appendix B: For Basesized models we searched for a learning
- "squad": {
- "num_train_epochs": [2]
- },
- "squad_v2": {
- "num_train_epochs": [2]
- },
- "glue_stsb": {
- "num_train_epochs": [10],
- },
- "glue_rte": {
- "num_train_epochs": [10],
- },
- "glue_wnli": {
- "num_train_epochs": [3],
- },
- "glue_mrpc": {
- "num_train_epochs": [3],
- },
- "glue_cola": {
- "num_train_epochs": [3],
- },
- "glue_sst2": {
- "num_train_epochs": [3],
- },
- "glue_qnli": {
- "num_train_epochs": [3],
- },
- "glue_mnli": {
- "num_train_epochs": [3],
- },
- "glue_qqp": {
- "num_train_epochs": [3],
- }
- }
- from ..result_analysis.azure_utils import JobID
- return get_space_union_and_unique(search_space_common, search_space_unique,
- [JobID.get_full_data_name(
- dataset_name_list,
- subdataset_name), model_size_type])
-
-
-def get_mobilebert_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices
- https://arxiv.org/pdf/2004.02984.pdf
- """
- # To finetune the pre-trained models, we search the optimization hyperparameters
- # in a search space including different batch sizes (16/32/48), learning
- # rates ((1-10) * e-5), and the number of epochs (2-10)
- search_space_common = {
- "learning_rate": [x * 1e-5 for x in range(1, 11)],
- "per_device_train_batch_size": [4, 8, 16, 32, 48],
- "num_train_epochs": [x for x in range(2, 11)],
- }
- search_space_unique = {}
- return get_space_union_and_unique(search_space_common, search_space_unique, [])
-
-
-def get_albert_space(model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- Hyperparameters for downstream tasks are shown in Table 14. We adapt these hyperparameters
- from Liu et al. (2019), Devlin et al. (2019), and Yang et al. (2019).
-
- LR BSZ ALBERT DR Classifier DR TS WS MSL
- CoLA 1.00E-05 16 0 0.1 5336 320 512
- STS 2.00E-05 16 0 0.1 3598 214 512
- SST-2 1.00E-05 32 0 0.1 20935 1256 512
- MNLI 3.00E-05 128 0 0.1 10000 1000 512
- QNLI 1.00E-05 32 0 0.1 33112 1986 512
- QQP 5.00E-05 128 0.1 0.1 14000 1000 512
- RTE 3.00E-05 32 0.1 0.1 800 200 512
- MRPC 2.00E-05 32 0 0.1 800 200 512
- WNLI 2.00E-05 16 0.1 0.1 2000 250 512
- SQuAD v1.1 5.00E-05 48 0 0.1 3649 365 384
- SQuAD v2.0 3.00E-05 48 0 0.1 8144 814 512
- RACE 2.00E-05 32 0.1 0.1 12000 1000 512
- """
- search_space_common = {
- }
- search_space_unique = {
- "glue_cola": {
- "learning_rate": [1e-5],
- "per_device_train_batch_size": [16],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [5336],
- "warmup_steps": [320],
- },
- "glue_stsb": {
- "learning_rate": [2e-5],
- "per_device_train_batch_size": [16],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [3598],
- "warmup_steps": [214],
- },
- "glue_sst2": {
- "learning_rate": [1e-5],
- "per_device_train_batch_size": [32],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [20935],
- "warmup_steps": [1256],
- },
- "glue_mnli": {
- "learning_rate": [3e-5],
- "per_device_train_batch_size": [128],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [10000],
- "warmup_steps": [1000],
- },
- "glue_qnli": {
- "learning_rate": [1e-5],
- "per_device_train_batch_size": [32],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [33112],
- "warmup_steps": [1986],
- },
- "glue_qqp": {
- "learning_rate": [5e-5],
- "per_device_train_batch_size": [128],
- "attention_probs_dropout_prob": [0.1],
- "classifier_dropout_prob": [0.1],
- "max_steps": [14000],
- "warmup_steps": [1000],
- },
- "glue_rte": {
- "learning_rate": [3e-5],
- "per_device_train_batch_size": [32],
- "attention_probs_dropout_prob": [0.1],
- "classifier_dropout_prob": [0.1],
- "max_steps": [800],
- "warmup_steps": [200],
- },
- "glue_mrpc": {
- "learning_rate": [2e-5],
- "per_device_train_batch_size": [32],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [800],
- "warmup_steps": [200],
- },
- "glue_wnli": {
- "learning_rate": [2e-5],
- "per_device_train_batch_size": [16],
- "attention_probs_dropout_prob": [0.1],
- "classifier_dropout_prob": [0.1],
- "max_steps": [2000],
- "warmup_steps": [250],
- },
- "squad": {
- "learning_rate": [5e-5],
- "per_device_train_batch_size": [48],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [3649],
- "warmup_steps": [365],
- },
- "squad_v2": {
- "learning_rate": [3e-5],
- "per_device_train_batch_size": [48],
- "attention_probs_dropout_prob": [0],
- "classifier_dropout_prob": [0.1],
- "max_steps": [8144],
- "warmup_steps": [814],
- },
- "race": {
- "learning_rate": [2e-5],
- "per_device_train_batch_size": [32],
- "attention_probs_dropout_prob": [0.1],
- "classifier_dropout_prob": [0.1],
- "max_steps": [12000],
- "warmup_steps": [1000],
- },
- }
-
- # To finetune the pre-trained models, we search the optimization hyperparameters
- # in a search space including different batch sizes (16/32/48), learning
- # rates ((1-10) * e-5), and the number of epochs (2-10)
- from ..result_analysis.azure_utils import JobID
- return get_space_union_and_unique(search_space_common, search_space_unique,
- [JobID.get_full_data_name(
- dataset_name_list,
- subdataset_name)])
diff --git a/flaml/nlp/hpo/grid_searchspace_auto.py b/flaml/nlp/hpo/grid_searchspace_auto.py
deleted file mode 100644
index 3e420bd6f..000000000
--- a/flaml/nlp/hpo/grid_searchspace_auto.py
+++ /dev/null
@@ -1,97 +0,0 @@
-from collections import OrderedDict
-
-from .get_grid_search_space import \
- (get_electra_space,
- get_bert_space,
- get_roberta_space,
- get_funnel_space,
- get_deberta_space,
- get_albert_space,
- get_longformer_space,
- get_mobilebert_space
- )
-
-GRID_SEARCH_SPACE_MAPPING = OrderedDict(
- [
- ("electra", get_electra_space),
- ("bert", get_bert_space),
- ("roberta", get_roberta_space),
- ("funnel", get_funnel_space),
- ("deberta", get_deberta_space),
- ("albert", get_albert_space),
- ("mobilebert", get_mobilebert_space),
- ("longformer", get_longformer_space)
- ]
-)
-
-HF_MODEL_LIST = [
- "bert",
- "roberta",
- "electra",
- "xlnet",
- "albert",
- "distilbert",
- "deberta",
- "mobilebert",
- "funnel"
-]
-
-
-class AutoGridSearchSpace:
- """
- This is a class for getting the recommended grid search space of a pre-trained LM that will be
- instantiated as one of the search spaces of the library when created with the
- `~flaml.nlp.hpo.AutoGridSearchSpace.from_model_and_dataset_name` method.
-
- This class cannot be instantiated directly using ``__init__()`` (throws an error).
- """
-
- def __init__(self):
- raise EnvironmentError(
- "AutoGridSearchSpace is designed to be instantiated "
- "using the `AutoGridSearchSpace.from_config_and_method_name(cls, model_type, model_size_type,"
- "dataset_name,subdataset_name = None,algo_mode = None)` methods."
- )
-
- @classmethod
- def from_model_and_dataset_name(cls,
- model_type,
- model_size_type,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None):
- """
- Instantiate one of the classes for getting the recommended grid search space of a pre-trained LM from
- the model type, model size type, dataset name, sub dataset name and algorithm mode
-
- Args:
- model_type:
- A string variable which is the model type, e.g. "electra"
-
- model_size_type:
- A string variable which is the size of the model, e.g., "small"
-
- dataset_name_list:
- A string variable which is the dataset name, e.g., "glue"
-
- subdataset_name:
- A string variable which is the sub dataset name,e.g., "rte"
-
- algo_mode:
- A string variable which is the algorithm mode for grid search, e.g., "gridbert"
-
- Example:
- >>> AutoGridSearchSpace.from_model_and_dataset_name("electra", "small", ["glue"], "rte", "grid")
-
- """
-
- if model_type in GRID_SEARCH_SPACE_MAPPING.keys():
- this_model_recommended_space = GRID_SEARCH_SPACE_MAPPING[model_type](
- model_size_type, dataset_name_list, subdataset_name, algo_mode)
- return this_model_recommended_space
- raise ValueError(
- "Unrecognized method {},{} for this kind of AutoGridSearchSpace: {}.\n"
- "Method name should be one of {}.".format(
- model_type, dataset_name_list, cls.__name__, ", ".join(GRID_SEARCH_SPACE_MAPPING.keys())
- )
- )
diff --git a/flaml/nlp/hpo/hpo_searchspace.py b/flaml/nlp/hpo/hpo_searchspace.py
deleted file mode 100644
index 7e0b440a8..000000000
--- a/flaml/nlp/hpo/hpo_searchspace.py
+++ /dev/null
@@ -1,269 +0,0 @@
-from collections import OrderedDict
-from .grid_searchspace_auto import AutoGridSearchSpace
-
-
-def hpo_space_custom(model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- """
- The 5 arguments here cannot be deleted, they need to be kept consistent with
- other functions in HPO_SEARCH_SPACE_MAPPING
- """
- assert "hpo_space" in custom_hpo_args
- custom_search_space = custom_hpo_args["hpo_space"]
- return custom_search_space
-
-
-def bounded_gridunion(model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- assert "bound" in custom_hpo_args
- gridunion_space = HPO_SEARCH_SPACE_MAPPING["uni"](model_type,
- model_size_type,
- dataset_name_list,
- subdataset_name,
- **custom_hpo_args)
- for each_key in custom_hpo_args["bound"].keys():
- if "u" in custom_hpo_args["bound"][each_key]:
- upper = custom_hpo_args["bound"][each_key]["u"]
- else:
- # TODO coverage
- upper = 100000
- if "l" in custom_hpo_args["bound"][each_key]:
- # TODO coverage
- lower = custom_hpo_args["bound"][each_key]["l"]
- else:
- lower = -100000
- original_space = sorted(gridunion_space[each_key])
- upper_id = len(original_space)
- for x in range(len(original_space)):
- if original_space[x] > upper:
- # TODO coverage
- upper_id = x
- break
- lower_id = 0
- for x in range(len(original_space) - 1, -1, -1):
- if original_space[x] < lower:
- lower_id = x
- break
- gridunion_space[each_key] = original_space[lower_id:upper_id]
- return gridunion_space
-
-
-def hpo_space_gridunion(model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- output_config = {}
- for each_model_type in ["bert", "roberta", "electra"]:
- # if each_model_type == model_type: continue
- this_config = AutoGridSearchSpace.from_model_and_dataset_name(
- each_model_type, model_size_type, dataset_name_list, subdataset_name, "hpo")
- from ..utils import merge_dicts
- output_config = merge_dicts(output_config, this_config)
- default_values = {}
- """
- adding the default configuration from transformers/training_args.py into hpo space
- """
- from transformers import TrainingArguments
- training_args = TrainingArguments(output_dir=".")
- for each_hp in output_config.keys():
- try:
- default_values[each_hp] = [getattr(training_args, each_hp)]
- except AttributeError:
- print("training args do not contain {}, passed".format(each_hp))
-
- output_config = merge_dicts(output_config, default_values)
-
- return output_config
-
-
-def hpo_space_gridunion_smoke_test(
- model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- return {
- "learning_rate": {"l": 1e-6, "u": 1e-3, "space": "log"},
- "num_train_epochs": [0.01],
- "per_device_train_batch_size": [2],
- "warmup_ratio": {"l": 0.0, "u": 0.3, "space": "linear"},
- "weight_decay": {"l": 0.0, "u": 0.3, "space": "linear"}
- }
-
-
-def hpo_space_generic(model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- output_config = {
- "learning_rate": {"l": 1e-6, "u": 1e-3, "space": "log"},
- "num_train_epochs": {"l": 1.0, "u": 10.0, "space": "log"},
- "per_device_train_batch_size": [4, 8, 16, 32, 48],
- "warmup_ratio": {"l": 0.0, "u": 0.3, "space": "linear"},
- "weight_decay": {"l": 0.0, "u": 0.3, "space": "linear"}
- }
- return output_config
-
-
-def hpo_space_generic_grid(model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- # TODO coverage
- output_config = {
- "learning_rate": [1e-5, 2e-5, 3e-5, 4e-5, 5e-5, 1e-4, 1.5e-4],
- "num_train_epochs": [3, 10],
- "per_device_train_batch_size": [16, 32],
- "warmup_ratio": [0, 0.06, 0.1],
- "weight_decay": [0, 0.1]
- }
- return output_config
-
-
-def hpo_space_small(model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- # TODO coverage
- config_json = AutoGridSearchSpace.from_model_and_dataset_name(
- model_type, model_size_type, dataset_name_list, subdataset_name, "hpo")
- output_config = {}
-
- for each_hp in config_json.keys():
- if each_hp == "learning_rate":
- if len(config_json[each_hp]) > 1:
- output_config[each_hp] = {"l": 3e-5, "u": 1.5e-4, "space": "log"}
- else:
- output_config[each_hp] = config_json[each_hp]
- elif each_hp == "num_train_epochs":
- output_config[each_hp] = {"l": 2.0, "u": 4.0, "space": "linear"}
- elif each_hp == "per_device_train_batch_size":
- output_config[each_hp] = [16, 32, 64]
- elif each_hp == "warmup_ratio":
- output_config[each_hp] = {"l": 0.0, "u": 0.2, "space": "linear"}
- elif each_hp == "weight_decay":
- output_config[each_hp] = {"l": 0.0, "u": 0.3, "space": "linear"}
- else:
- output_config[each_hp] = config_json[each_hp]
-
- return output_config
-
-
-def hpo_space_grid(model_type=None,
- model_size_type=None,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- return AutoGridSearchSpace.from_model_and_dataset_name(model_type,
- model_size_type,
- dataset_name_list,
- subdataset_name,
- algo_mode
- )
-
-
-HPO_SEARCH_SPACE_MAPPING = OrderedDict(
- [
- ("grid", hpo_space_grid),
- ("uni", hpo_space_gridunion),
- ("gnr", hpo_space_generic),
- ("uni_test", hpo_space_gridunion_smoke_test),
- ("cus", hpo_space_custom),
- ("buni", bounded_gridunion),
- ]
-)
-
-
-class AutoHPOSearchSpace:
- """
- This is a class for getting the hpo search space based on the search space mode
- (a string variable) instantiated as one of the HPO search spaces of the library when
- created with the `~flaml.nlp.hpo.AutoHPOSearchSpace.from_model_and_dataset_name` method.
-
- This class cannot be instantiated directly using ``__init__()`` (throws an error).
- """
-
- def __init__(self):
- raise EnvironmentError(
- "AutoHPOSearchSpace is designed to be instantiated "
- "using the `AutoHPOSearchSpace.from_config_and_method_name(cls, hpo_searchspace_name,"
- "model_type,model_size_type,dataset_name,subdataset_name=None,**custom_hpo_args)` methods."
- )
-
- @classmethod
- def from_model_and_dataset_name(cls,
- hpo_searchspace_mode,
- model_type,
- model_size_type,
- dataset_name_list: list = None,
- subdataset_name=None,
- algo_mode=None,
- **custom_hpo_args):
- """
- Instantiate one of the classes for getting the hpo search space from the search space name, model type,
- model size type, dataset name and sub dataset name
-
- Args:
-
- hpo_searchspace_mode:
- A string variable which is the mode of the hpo search space, it must be chosen from the following options:
- - uni: the union of BERT, RoBERTa and Electra's grid configs
- - grid: the recommended grid config of the LM specified in jobconfig.pre
- - gnr: the generic continuous search space
- - uni_test: the search space for smoke test
- - cus: user customized search space, specified in the "hpo_space" argument in AutoTransformers.fit
- - buni: bounded grid union search space
-
- model_type:
- A string variable which is the type of the model, e.g., "electra"
-
- model_size_type:
- A string variable which is the type of the model size, e.g., "small"
-
- dataset_name:
- A string variable which is the dataset name, e.g., "glue"
-
- subdataset_name:
- A string variable which is the sub dataset name,e.g., "rte"
-
- custom_hpo_args:
- Any additional keyword argument to be used for the function for the HPO search space
-
- Example:
- >>> AutoHPOSearchSpace.from_model_and_dataset_name("uni", "electra", "small", ["glue"], "rte", "hpo")
- """
-
- if hpo_searchspace_mode in HPO_SEARCH_SPACE_MAPPING.keys():
- hpo_space = HPO_SEARCH_SPACE_MAPPING[hpo_searchspace_mode](
- model_type,
- model_size_type,
- dataset_name_list,
- subdataset_name,
- algo_mode,
- **custom_hpo_args)
- return hpo_space
- raise ValueError(
- "Unrecognized method {},{} for this kind of AutoHPOSearchSpace: {}.\n"
- "Method name should be one of {}.".format(
- hpo_searchspace_mode, dataset_name_list, cls.__name__,
- ", ".join(HPO_SEARCH_SPACE_MAPPING.keys())
- )
- )
diff --git a/flaml/nlp/hpo/scheduler_auto.py b/flaml/nlp/hpo/scheduler_auto.py
deleted file mode 100644
index eca680de1..000000000
--- a/flaml/nlp/hpo/scheduler_auto.py
+++ /dev/null
@@ -1,49 +0,0 @@
-from collections import OrderedDict
-from ray.tune.schedulers import ASHAScheduler, HyperBandScheduler
-
-SCHEDULER_MAPPING = OrderedDict(
- [
- ("None", None),
- ("asha", ASHAScheduler),
- ("hb", HyperBandScheduler),
- ]
-)
-
-
-class AutoScheduler:
- """
- This is a class for getting the scheduler based on the scheduler name
- (a string variable) instantiated as one of the schedulers of the library when
- created with the `~flaml.nlp.hpo.AutoScheduler.from_scheduler_name` method.
-
- This class cannot be instantiated directly using ``__init__()`` (throws an error).
- """
-
- def __init__(self):
- raise EnvironmentError(
- "AutoScheduler is designed to be instantiated "
- "using the `AutoScheduler.from_scheduler_name(cls, scheduler_name, **kwargs)` methods."
- )
-
- @classmethod
- def from_scheduler_name(cls, scheduler_name, **kwargs):
- """
- Instantiate one of the schedulers using the scheduler names
-
- Args:
- scheduler_name:
- A string variable for the scheduler name
-
- Example:
- >>> AutoScheduler.from_scheduler_name("asha")
- """
- if scheduler_name in SCHEDULER_MAPPING.keys():
- if SCHEDULER_MAPPING[scheduler_name] is None:
- return None
- return SCHEDULER_MAPPING[scheduler_name](**kwargs)
- raise ValueError(
- "Unrecognized scheduler {} for this kind of AutoScheduler: {}.\n"
- "Scheduler name should be one of {}.".format(
- scheduler_name, cls.__name__, ", ".join(SCHEDULER_MAPPING.keys())
- )
- )
diff --git a/flaml/nlp/hpo/searchalgo_auto.py b/flaml/nlp/hpo/searchalgo_auto.py
deleted file mode 100644
index 441150026..000000000
--- a/flaml/nlp/hpo/searchalgo_auto.py
+++ /dev/null
@@ -1,230 +0,0 @@
-import itertools
-from collections import OrderedDict
-
-import ray
-from ray.tune.suggest.optuna import OptunaSearch
-
-from flaml import CFO, BlendSearch
-
-SEARCH_ALGO_MAPPING = OrderedDict(
- [
- ("optuna", OptunaSearch),
- ("cfo", CFO),
- ("bs", BlendSearch),
- ("grid", None),
- ("gridbert", None),
- ("rs", None),
- ]
-)
-
-
-class AutoSearchAlgorithm:
- """
- This is a class for getting the search algorithm based on the search algorithm name
- (a string variable) instantiated as one of the algorithms of the library when
- created with the `~flaml.nlp.hpo.AutoSearchAlgorithm.from_method_name` method.
-
- This class cannot be instantiated directly using ``__init__()`` (throws an error).
- """
-
- def __init__(self):
- raise EnvironmentError(
- "AutoSearchAlgorithm is designed to be instantiated "
- "using the `AutoSearchAlgorithm.from_method_name(cls, search_algo_name, search_algo_args_mode,"
- " hpo_search_space, **custom_hpo_args)` methods."
- )
-
- @classmethod
- def from_method_name(
- cls,
- search_algo_name,
- search_algo_args_mode,
- hpo_search_space,
- time_budget,
- metric_name,
- metric_mode_name,
- **custom_hpo_args
- ):
- """
- Instantiating one of the search algorithm classes based on the search algorithm name, search algorithm
- argument mode, hpo search space and other keyword args
-
- Args:
- search_algo_name:
- A string variable that specifies the search algorithm name, e.g., "bs"
-
- search_algo_args_mode:
- A string variable that specifies the mode for the search algorithm args, e.g., "dft" means
- initializing using the default mode
-
- hpo_search_space:
- The hpo search space
-
- custom_hpo_args:
- The customized arguments for the search algorithm (specified by user)
-
- Example:
- >>> from flaml.nlp.hpo.hpo_searchspace import AutoHPOSearchSpace
- >>> search_space_hpo=AutoHPOSearchSpace.from_model_and_dataset_name("uni", "electra", "small", ["glue"], "rte")
- >>> search_algo = AutoSearchAlgorithm.from_method_name("bs", "cus", search_space_hpo,
- {"points_to_evaluate": [{"learning_rate": 1e-5, "num_train_epochs": 10}])
- """
-
- assert (
- hpo_search_space
- ), "hpo_search_space needs to be specified for calling AutoSearchAlgorithm.from_method_name"
- if not search_algo_name:
- # TODO coverage
- search_algo_name = "grid"
- if search_algo_name in SEARCH_ALGO_MAPPING.keys():
- if SEARCH_ALGO_MAPPING[search_algo_name] is None:
- # TODO coverage
- return None
- """
- filtering the customized args for hpo from custom_hpo_args, keep those
- which are in the input variable name list of the constructor of
- the algorithm, remove those which does not appear in the input variables
- of the constructor function
- """
- this_search_algo_kwargs = None
- allowed_arguments = SEARCH_ALGO_MAPPING[
- search_algo_name
- ].__init__.__code__.co_varnames
- custom_hpo_args["time_budget_s"] = time_budget
- allowed_custom_args = {
- key: custom_hpo_args[key]
- for key in custom_hpo_args.keys()
- if key in allowed_arguments
- }
-
- """
- If the search_algo_args_mode is "dft", set the args to the default args, e.g.,the default args for
- BlendSearch is "low_cost_partial_config": {"num_train_epochs": min_epoch,"per_device_train_batch_size"
- : max(hpo_search_space["per_device_train_batch_size"].categories)},
- """
- if search_algo_args_mode == "dft":
- # TODO coverage
- this_search_algo_kwargs = DEFAULT_SEARCH_ALGO_ARGS_MAPPING[
- search_algo_name
- ](
- "dft",
- metric_name,
- metric_mode_name,
- hpo_search_space=hpo_search_space,
- **allowed_custom_args
- )
- elif search_algo_args_mode == "cus":
- this_search_algo_kwargs = DEFAULT_SEARCH_ALGO_ARGS_MAPPING[
- search_algo_name
- ](
- "cus",
- metric_name,
- metric_mode_name,
- hpo_search_space=hpo_search_space,
- **allowed_custom_args
- )
-
- """
- returning the hpo algorithm with the arguments
- """
- search_algo = SEARCH_ALGO_MAPPING[search_algo_name](
- **this_search_algo_kwargs
- )
- if search_algo_name == "bs":
- search_algo.set_search_properties()
- return search_algo
- raise ValueError(
- "Unrecognized method {} for this kind of AutoSearchAlgorithm: {}.\n"
- "Method name should be one of {}.".format(
- search_algo_name, cls.__name__, ", ".join(SEARCH_ALGO_MAPPING.keys())
- )
- )
-
- @staticmethod
- def grid2list(grid_config):
- # TODO coverage
- key_val_list = [
- [(key, each_val) for each_val in val_list["grid_search"]]
- for (key, val_list) in grid_config.items()
- ]
- config_list = [dict(x) for x in itertools.product(*key_val_list)]
- return config_list
-
-
-def get_search_algo_args_optuna(
- search_args_mode,
- metric_name,
- metric_mode_name,
- hpo_search_space=None,
- **custom_hpo_args
-):
- # TODO coverage
- return {}
-
-
-def default_search_algo_args_bs(
- search_args_mode,
- metric_name,
- metric_mode_name,
- hpo_search_space=None,
- time_budget_s=None,
- **custom_hpo_args
-):
- assert (
- hpo_search_space
- ), "hpo_search_space needs to be specified for calling AutoSearchAlgorithm.from_method_name"
- if "num_train_epochs" in hpo_search_space and isinstance(
- hpo_search_space["num_train_epochs"], ray.tune.sample.Categorical
- ):
- min_epoch = min(hpo_search_space["num_train_epochs"].categories)
- else:
- # TODO coverage
- assert isinstance(hpo_search_space["num_train_epochs"], ray.tune.sample.Float)
- min_epoch = hpo_search_space["num_train_epochs"].lower
- default_search_algo_args = {
- "low_cost_partial_config": {
- "num_train_epochs": min_epoch,
- "per_device_train_batch_size": max(
- hpo_search_space["per_device_train_batch_size"].categories
- ),
- },
- "space": hpo_search_space,
- "metric": metric_name,
- "mode": metric_mode_name,
- "time_budget_s": time_budget_s,
- }
- if search_args_mode == "cus":
- default_search_algo_args.update(custom_hpo_args)
- return default_search_algo_args
-
-
-def default_search_algo_args_grid_search(
- search_args_mode,
- metric_name,
- metric_mode_name,
- hpo_search_space=None,
- **custom_hpo_args
-):
- # TODO coverage
- return {}
-
-
-def default_search_algo_args_random_search(
- search_args_mode,
- metric_name,
- metric_mode_name,
- hpo_search_space=None,
- **custom_hpo_args
-):
- # TODO coverage
- return {}
-
-
-DEFAULT_SEARCH_ALGO_ARGS_MAPPING = OrderedDict(
- [
- ("optuna", get_search_algo_args_optuna),
- ("cfo", default_search_algo_args_bs),
- ("bs", default_search_algo_args_bs),
- ("grid", default_search_algo_args_grid_search),
- ]
-)
diff --git a/flaml/nlp/huggingface/__init__.py b/flaml/nlp/huggingface/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/flaml/nlp/huggingface/switch_head_auto.py b/flaml/nlp/huggingface/switch_head_auto.py
deleted file mode 100644
index 8eef47a67..000000000
--- a/flaml/nlp/huggingface/switch_head_auto.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from collections import OrderedDict
-
-import transformers
-
-if transformers.__version__.startswith("3"):
- from transformers.modeling_electra import ElectraClassificationHead
- from transformers.modeling_roberta import RobertaClassificationHead
-else:
- from transformers.models.electra.modeling_electra import ElectraClassificationHead
- from transformers.models.roberta.modeling_roberta import RobertaClassificationHead
-
-MODEL_CLASSIFICATION_HEAD_MAPPING = OrderedDict(
- [
- ("electra", ElectraClassificationHead),
- ("roberta", RobertaClassificationHead),
- ]
-)
-
-
-class AutoSeqClassificationHead:
- """
- This is a class for getting classification head class based on the name of the LM
- instantiated as one of the ClassificationHead classes of the library when
- created with the `~flaml.nlp.huggingface.AutoSeqClassificationHead.from_model_type_and_config` method.
-
- This class cannot be instantiated directly using ``__init__()`` (throws an error).
- """
-
- def __init__(self):
- raise EnvironmentError(
- "AutoSeqClassificationHead is designed to be instantiated "
- "using the `AutoSeqClassificationHead.from_model_type_and_config(cls, model_type, config)` methods."
- )
-
- @classmethod
- def from_model_type_and_config(cls, model_type, config):
- """
- Instantiate one of the classification head classes from the mode_type and model configuration.
-
- Args:
- model_type:
- A string, which desribes the model type, e.g., "electra"
- config (:class:`~transformers.PretrainedConfig`):
- The huggingface class of the model's configuration:
-
- Examples::
- >>> from transformers import AutoConfig
- >>> model_config = AutoConfig.from_pretrained("google/electra-base-discriminator")
- >>> AutoSeqClassificationHead.from_model_type_and_config("electra", model_config)
- """
- if model_type in MODEL_CLASSIFICATION_HEAD_MAPPING.keys():
- return MODEL_CLASSIFICATION_HEAD_MAPPING[model_type](config)
- raise ValueError(
- "Unrecognized configuration class {} for class {}.\n"
- "Model type should be one of {}.".format(
- config.__class__, cls.__name__, ", ".join(MODEL_CLASSIFICATION_HEAD_MAPPING.keys())
- )
- )
diff --git a/flaml/nlp/huggingface/trainer.py b/flaml/nlp/huggingface/trainer.py
deleted file mode 100644
index d1560968a..000000000
--- a/flaml/nlp/huggingface/trainer.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import os
-
-try:
- from transformers import Trainer as TFTrainer
-except ImportError:
- TFTrainer = object
-
-
-class TrainerForAutoTransformers(TFTrainer):
-
- def evaluate(self,
- eval_dataset=None):
- """
- Overriding transformers.Trainer.evaluate by saving state with save_state
-
- Args:
- eval_dataset:
- the dataset to be evaluated
- """
- # TODO coverage
- from ray import tune
-
- eval_dataloader = self.get_eval_dataloader(eval_dataset)
- output = self.prediction_loop(
- eval_dataloader, description="Evaluation")
- self.log(output.metrics)
-
- self.save_state()
-
- for key in list(output.metrics.keys()):
- if key.startswith("eval_"):
- output.metrics[key[5:]] = output.metrics[key]
- tune.report(**output.metrics)
-
- return output.metrics
-
- def save_state(self):
- """
- Overriding transformers.Trainer.save_state. It is only through saving
- the states can best_trial.get_best_checkpoint return a non-empty value.
- """
- # TODO coverage
- import torch
- from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
- from ray import tune
-
- with tune.checkpoint_dir(step=self.state.global_step) as checkpoint_dir:
- self.args.output_dir = checkpoint_dir
- # This is the directory name that Huggingface requires.
- output_dir = os.path.join(
- self.args.output_dir,
- f"{PREFIX_CHECKPOINT_DIR}-{self.state.global_step}")
- self.save_model(output_dir)
- torch.save(self.optimizer.state_dict(),
- os.path.join(output_dir, "optimizer.pt"))
- torch.save(self.lr_scheduler.state_dict(),
- os.path.join(output_dir, "scheduler.pt"))
-
- @staticmethod
- def convert_num_train_epochs_to_max_steps(
- num_train_epochs: int,
- num_train_examples: int,
- per_device_train_batch_size: int,
- device_count: int):
- return int(num_train_epochs * num_train_examples / per_device_train_batch_size / device_count)
-
- @staticmethod
- def convert_max_steps_to_num_train_epochs(
- max_steps: int,
- num_train_examples: int,
- per_device_train_batch_size: int,
- device_count: int):
- return float(max_steps * per_device_train_batch_size * device_count) / num_train_examples
-
- @staticmethod
- def convert_warmup_ratio_to_warmup_steps(
- warmup_ratio,
- max_steps=None,
- num_train_epochs=None,
- num_train_examples=None,
- per_device_train_batch_size=None,
- device_count=None):
- if max_steps:
- return int(warmup_ratio * max_steps)
- # TODO coverage
- max_steps = TrainerForAutoTransformers.convert_num_train_epochs_to_max_steps(
- num_train_epochs,
- num_train_examples,
- per_device_train_batch_size,
- device_count)
- return int(warmup_ratio * max_steps)
-
- @staticmethod
- def convert_warmup_steps_to_warmup_ratio(
- warmup_steps: int,
- num_train_epochs: int,
- num_train_examples: int,
- per_device_train_batch_size: int,
- device_count: int):
- max_steps = TrainerForAutoTransformers.convert_num_train_epochs_to_max_steps(
- num_train_epochs,
- num_train_examples,
- per_device_train_batch_size,
- device_count)
- return float(warmup_steps / max_steps)
diff --git a/flaml/nlp/result_analysis/__init__.py b/flaml/nlp/result_analysis/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/flaml/nlp/result_analysis/azure_utils.py b/flaml/nlp/result_analysis/azure_utils.py
deleted file mode 100644
index 7aee85ad9..000000000
--- a/flaml/nlp/result_analysis/azure_utils.py
+++ /dev/null
@@ -1,719 +0,0 @@
-import re
-import pathlib
-import os
-from datetime import datetime
-from dataclasses import dataclass, field
-import json
-from typing import Tuple, List, Union, Optional
-import argparse
-
-
-class ConfigScore:
- trial_id: str = field(default=None)
- start_time: float = field(default=None)
- last_update_time: float = field(default=None)
- config: dict = field(default=None)
- metric_score: dict = field(default=None)
- time_stamp: float = field(default=None)
-
- def __init__(self,
- trial_id: str = None,
- start_time: float = None,
- last_update_time: float = None,
- config: dict = None,
- metric_score: dict = None,
- time_stamp: float = None
- ):
- self.trial_id = trial_id
- self.start_time = start_time
- self.last_update_time = last_update_time
- self.config = config
- self.metric_score = metric_score
- self.time_stamp = time_stamp
-
-
-class ConfigScoreList:
-
- def __init__(self,
- config_score_list: List[ConfigScore],
- jobid_config=None,
- blob_file=None,
- ):
- self._config_score_list = config_score_list
- self._blob_file = blob_file
- self._jobid_config = jobid_config
-
- def sorted(self, sort_method="unsorted", metric_mode="max"):
- if sort_method == "unsorted":
- self._config_score_list = self._config_score_list
- elif sort_method == "sort_time":
- self._config_score_list = sorted(self._config_score_list, key=lambda x: x.start_time, reverse=False)
- else:
- self._config_score_list = sorted(self._config_score_list,
- key=lambda x: getattr(x, "metric_score")
- [metric_mode], reverse=True)
-
- def get_best_config(self,
- metric_mode="max"):
- return max(self._config_score_list, key=lambda x: getattr(x, "metric_score")[metric_mode])
-
-
-@dataclass
-class JobID:
- """
- The class for specifying the config of a job, includes the following fields:
-
- dat:
- A list which is the dataset name
- subdat:
- A string which is the sub dataset name
- mod:
- A string which is the module, e.g., "grid", "hpo"
- spa:
- A string which is the space mode, e.g., "uni", "gnr"
- arg:
- A string which is the mode for setting the input argument of a search algorithm, e.g., "cus", "dft"
- alg:
- A string which is the search algorithm name
- pru:
- A string which is the scheduler name
- pre_full:
- A string which is the full name of the pretrained language model
- pre:
- A string which is the abbreviation of the pretrained language model
- presz:
- A string which is the size of the pretrained language model
- spt:
- A string which is the resplit mode, e.g., "ori", "rspt"
- rep:
- An integer which is the repetition id
- sddt:
- An integer which is the seed for data shuffling in the resplit mode
- sdhf:
- An integer which is the seed for transformers
- """
- dat: list = field(default=None)
- subdat: str = field(default=None)
- mod: str = field(default=None)
- spa: str = field(default=None)
- arg: str = field(default=None)
- alg: str = field(default=None)
- pru: str = field(default=None)
- pre_full: str = field(default=None)
- pre: str = field(default=None)
- presz: str = field(default=None)
- spt: str = field(default=None)
- rep: int = field(default=0)
- sddt: int = field(default=None)
- sdhf: int = field(default=None)
-
- def __init__(self,
- console_args=None):
- if console_args:
- self.set_jobid_from_console_args(console_args)
-
- def set_unittest_config(self):
- """
- set the JobID config for unit test
- """
- self.dat = ["glue"]
- self.subdat = "mrpc"
- self.mod = "hpo"
- self.spa = "uni_test"
- self.arg = "cus"
- self.alg = "bs"
- self.pru = "None"
- self.pre_full = "google/mobilebert-uncased"
- self.pre = "mobilebert"
- self.presz = "small"
- self.spt = "rspt"
- self.rep = 0
- self.sddt = 43
- self.sdhf = 42
-
- def is_match(self, partial_jobid):
- """Return a boolean variable whether the current object matches the partial jobid defined in partial_jobid.
-
- Example:
-
- .. code-block:: python
-
- self = JobID(dat = ['glue'], subdat = 'cola', mod = 'bestnn', spa = 'buni', arg = 'cus', alg = 'bs',
- pru = 'None', pre = 'funnel', presz = 'xlarge', spt = 'rspt', rep = 0, sddt = 43, sdhf = 42)
-
- partial_jobid1 = JobID(dat = ['glue'],
- subdat = 'cola',
- mod = 'hpo')
-
- partial_jobid2 = JobID(dat = ['glue'],
- subdat = 'cola',
- mod = 'bestnn')
-
- return False for partial_jobid1 and True for partial_jobid2
- """
- is_not_match = False
- for key, val in partial_jobid.__dict__.items():
- if val is None:
- continue
- if getattr(self, key) != val:
- is_not_match = True
- return not is_not_match
-
- def to_wandb_string(self):
- """
- Preparing for the job ID for wandb
- """
- field_dict = self.__dict__
- keytoval_str = "_".join([JobID.dataset_list_to_str(field_dict[key])
- if type(field_dict[key]) == list
- else str(field_dict[key])
- for key in field_dict.keys() if not key.endswith("_full")])
- return keytoval_str
-
- def to_jobid_string(self):
- """
- Convert the current JobID into a blob name string which contains all the fields
- """
- list_keys = list(JobID.__dataclass_fields__.keys())
- field_dict = self.__dict__
- keytoval_str = "_".join([key + "=" + JobID.dataset_list_to_str(field_dict[key])
- if type(field_dict[key]) == list
- else key + "=" + str(field_dict[key])
- for key in list_keys if not key.endswith("_full")])
- return keytoval_str
-
- def to_partial_jobid_string(self):
- """
- Convert the current JobID into a blob name string which only contains the fields whose values are not "None"
- """
- list_keys = list(JobID.__dataclass_fields__.keys())
- field_dict = self.__dict__ # field_dict contains fields whose values are not None
- keytoval_str = "_".join([key + "=" + JobID.dataset_list_to_str(field_dict[key])
- if type(field_dict[key]) == list
- else key + "=" + str(field_dict[key])
- for key in list_keys if key in field_dict.keys()])
- return keytoval_str
-
- @staticmethod
- def blobname_to_jobid_dict(keytoval_str):
- """
- Converting an azure blobname to a JobID config,
- e.g., blobname = "dat=glue_subdat=cola_mod=bestnn_spa=buni_arg=cus_
- alg=bs_pru=None_pre=funnel_presz=xlarge_spt=rspt_rep=0.json"
-
- the converted jobid dict = {dat = ['glue'], subdat = 'cola', mod = 'bestnn',
- spa = 'buni', arg = 'cus', alg = 'bs', pru = 'None',
- pre = 'funnel', presz = 'xlarge', spt = 'rspt',
- rep = 0, sddt = 43, sdhf = 42)
- """
- field_keys = [key for key in list(JobID.__dataclass_fields__.keys()) if not key.endswith("_full")]
- regex_expression = ".*"
- is_first = True
- for key in field_keys:
- if is_first:
- prefix = ""
- is_first = False
- else:
- prefix = "_"
- if key.startswith("sd") or key.startswith("var"):
- regex_expression += "(" + prefix + key + "=(?P<" + key + ">[^_]*))?"
- else:
- regex_expression += prefix + key + "=(?P<" + key + ">[^_]*)"
- regex_expression += ".(json|zip)"
- result = re.search(regex_expression, keytoval_str)
- if result:
- result_dict = {}
- for key in field_keys:
- if key == "dat":
- result_dict[key] = [result.group(key)]
- elif key == "rep":
- try:
- try:
- result_dict[key] = int(result.group(key))
- except IndexError:
- print("No group {} in the regex result".format(key))
- result_dict[key] = -1
- except ValueError:
- print("Cannot parse integer {}".format(result.group(key)))
- result_dict[key] = -1
- else:
- result_dict[key] = result.group(key)
- return result_dict
- else:
- return None
-
- @staticmethod
- def dataset_list_to_str(dataset_name, key="dat"):
- if isinstance(dataset_name, list):
- return "-".join(dataset_name)
- else:
- return dataset_name
-
- def set_jobid_from_arg_list(self,
- **jobid_list
- ):
- """
- Set the jobid from a dict object
- """
- for key in jobid_list.keys():
- assert key in JobID.__dataclass_fields__.keys()
- setattr(self, key, jobid_list[key])
- if self.mod == "grid":
- self.alg = "grid"
-
- @staticmethod
- def convert_blobname_to_jobid(blobname):
- """
- Converting a blobname string to a JobID object
- """
- jobconfig_dict = JobID.blobname_to_jobid_dict(blobname)
- if jobconfig_dict:
- jobconfig = JobID()
- jobconfig.set_jobid_from_arg_list(**jobconfig_dict)
- return jobconfig
- else:
- return None
-
- @staticmethod
- def get_full_data_name(dataset_name: Union[list, str], subdataset_name=None):
- """
- Convert a dataset name and sub dataset name to a full dataset name
- """
- if isinstance(dataset_name, list):
- full_dataset_name = JobID.dataset_list_to_str(dataset_name)
- else:
- full_dataset_name = dataset_name
- if subdataset_name:
- full_dataset_name = full_dataset_name + "_" + subdataset_name
- return full_dataset_name
-
- def get_jobid_full_data_name(self):
- """
- Get the full dataset name of the current JobID object
- """
- return JobID.get_full_data_name(JobID.dataset_list_to_str(self.dat), self.subdat)
-
- @staticmethod
- def _extract_model_type_with_keywords_match(pre_full):
- from ..hpo.grid_searchspace_auto import HF_MODEL_LIST
- matched_model_type = []
- for each_model_type in HF_MODEL_LIST:
- if each_model_type in pre_full:
- matched_model_type.append(each_model_type)
- assert len(matched_model_type) > 0
- return max(enumerate(matched_model_type), key=lambda x: len(x[1]))[1]
-
- @staticmethod
- def extract_model_type(full_model_name):
- from transformers import AutoConfig
- model_config = AutoConfig.from_pretrained(full_model_name)
- config_json_file = model_config.get_config_dict(full_model_name)[0]
- try:
- model_type = config_json_file["model_type"]
- except KeyError:
- print("config_json_file does not contain model_type, re-extracting with keywords matching")
- model_type = JobID._extract_model_type_with_keywords_match(full_model_name)
- return model_type
-
- @staticmethod
- def get_attrval_from_arg_or_dict(console_args: Union[argparse.ArgumentParser, dict], each_key):
- if type(console_args) == argparse.Namespace:
- return getattr(console_args, each_key)
- else:
- return console_args[each_key]
-
- def set_jobid_from_console_args(self, console_args: Union[argparse.ArgumentParser, dict]):
- from ..utils import pretrained_model_size_format_check, dataset_subdataset_name_format_check
- console_to_jobid_key_mapping = {
- "pretrained_model_size": "pre",
- "dataset_subdataset_name": "dat",
- "algo_mode": "mod",
- "space_mode": "spa",
- "search_alg_args_mode": "arg",
- "algo_name": "alg",
- "pruner": "pru",
- "resplit_mode": "spt",
- "rep_id": "rep",
- "seed_data": "sddt",
- "seed_transformers": "sdhf",
- }
- for each_key in console_to_jobid_key_mapping.keys():
- try:
- try:
- if each_key == "dataset_subdataset_name":
- dataset_subdataset_name_format_check(JobID.get_attrval_from_arg_or_dict(console_args, each_key))
- self.dat = JobID.get_attrval_from_arg_or_dict(console_args, each_key).split(":")[0].split(",")
- self.subdat = JobID.get_attrval_from_arg_or_dict(console_args, each_key).split(":")[1]
- elif each_key == "pretrained_model_size":
- pretrained_model_size_format_check(JobID.get_attrval_from_arg_or_dict(console_args, each_key))
- self.pre_full = JobID.get_attrval_from_arg_or_dict(console_args, each_key).split(":")[0]
- self.pre = JobID.extract_model_type(self.pre_full)
- self.presz = JobID.get_attrval_from_arg_or_dict(console_args, each_key).split(":")[1]
- else:
- jobid_key = console_to_jobid_key_mapping[each_key]
- attrval = JobID.get_attrval_from_arg_or_dict(console_args, each_key)
- setattr(self, jobid_key, attrval)
- except AttributeError:
- print("console_args has no attribute {}, continue".format(each_key))
- continue
- except KeyError:
- print("console_args has no attribute {}, continue".format(each_key))
- continue
- if self.mod == "grid":
- # TODO coverage
- self.alg = "grid"
-
-
-class AzureUtils:
-
- def __init__(self,
- root_log_path=None,
- azure_key_path=None,
- data_root_dir=None,
- autohf=None,
- jobid_config=None):
- ''' This class is for saving the output files (logs, predictions) for HPO, uploading it to an azure storage
- blob, and performing analysis on the saved blobs. To use the cloud storage, you need to specify a key
- and upload the output files to azure. For example, when running jobs in a cluster, this class can
- help you store all the output files in the same place. If a key is not specified, this class will help you
- save the files locally but not uploading to the cloud. After the outputs are uploaded, you can use this
- class to perform analysis on the uploaded blob files.
-
- Examples:
-
- Example 1 (saving and uploading):
-
- validation_metric, analysis = autohf.fit(**autohf_settings) # running HPO
- predictions, test_metric = autohf.predict()
-
- azure_utils = AzureUtils(root_log_path="logs_test/",
- autohf=autohf,
- azure_key_path="../../")
- # passing the azure blob key from key.json under azure_key_path
-
- azure_utils.write_autohf_output(valid_metric=validation_metric,
- predictions=predictions,
- duration=autohf.last_run_duration)
- # uploading the output to azure cloud, which can be used for analysis afterwards
-
- Example 2 (analysis):
-
- jobid_config = JobID()
- jobid_config.mod = "grid"
- jobid_config.pre = "funnel"
- jobid_config.presz = "xlarge"
-
- azure_utils = AzureUtils(root_log_path= "logs_test/",
- azure_key_path = "../../",
- jobid_config=jobid_config)
-
- # continue analyzing all files in azure blob that matches jobid_config
-
- Args:
- root_log_path:
- The local root log folder name, e.g., root_log_path="logs_test/" will create a directory
- "logs_test/" locally
-
- azure_key_path:
- The path for storing the azure keys. The azure key, and container name are stored in a local file
- azure_key_path/key.json. The key_path.json file should look like this:
-
- {
- "container_name": "container_name",
- "azure_key": "azure_key",
- }
-
- To find out the container name and azure key of your blob, please refer to:
- https://docs.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-portal
-
- If the container name and azure key are not specified, the output will only be saved locally,
- not synced to azure blob.
-
- data_root_dir:
- The directory for outputing the predictions, e.g., packing the predictions into a .zip file for
- uploading to the glue website
-
- autohf:
- The AutoTransformers object, which contains the output of an HPO run. AzureUtils will save the
- output (analysis results, predictions) from AzureTransformers.
-
- jobid_config:
- The jobid config for analysis. jobid_config specifies the jobid config of azure blob files
- to be analyzed, if autohf is specified, jobid_config will be overwritten by autohf.jobid_config
- '''
- if root_log_path:
- self.root_log_path = root_log_path
- else:
- self.root_log_path = "logs_azure/"
- if autohf is not None:
- self.jobid = autohf.jobid_config
- else:
- # TODO coverage
- assert jobid_config is not None, "jobid_config must be passed either through autohf.jobid_config" \
- " or jobid_config"
- self.jobid = jobid_config
- self.data_root_dir = data_root_dir
- self.autohf = autohf
- if azure_key_path:
- azure_key, container_name = AzureUtils.get_azure_key(azure_key_path)
- self._container_name = container_name
- self._azure_key = azure_key
- else:
- self._container_name = self._azure_key = ""
-
- @staticmethod
- def get_azure_key(key_path):
- try:
- try:
- key_json = json.load(open(os.path.join(key_path, "key.json"), "r"))
- azure_key = key_json["azure_key"]
- azure_container_name = key_json["container_name"]
- return azure_key, azure_container_name
- except FileNotFoundError:
- print("Your output will not be synced to azure because key.json is not found under key_path")
- return "", ""
- except KeyError:
- print("Your output will not be synced to azure because azure key and container name are not specified")
- return "", ""
-
- def _get_complete_connection_string(self):
- try:
- return "DefaultEndpointsProtocol=https;AccountName=docws5141197765;AccountKey=" \
- + self._azure_key + ";EndpointSuffix=core.windows.net"
- except AttributeError:
- return "DefaultEndpointsProtocol=https;AccountName=docws5141197765;AccountKey=" \
- ";EndpointSuffix=core.windows.net"
-
- def _init_azure_clients(self):
- try:
- from azure.storage.blob import ContainerClient
- connection_string = self._get_complete_connection_string()
- try:
- container_client = ContainerClient.from_connection_string(conn_str=connection_string,
- container_name=self._container_name)
- return container_client
- except ValueError:
- print("Your output will not be synced to azure because azure key and container name are not specified")
- return None
- except ImportError:
- print("Your output will not be synced to azure because azure-blob-storage is not installed")
-
- def _init_blob_client(self,
- local_file_path):
- try:
- from azure.storage.blob import BlobServiceClient
-
- connection_string = self._get_complete_connection_string()
- blob_service_client = BlobServiceClient.from_connection_string(connection_string)
- try:
- blob_client = blob_service_client.get_blob_client(container=self._container_name, blob=local_file_path)
- return blob_client
- except ValueError:
- print("Your output will not be synced to azure because azure key and container name are not specified")
- return None
- except ImportError:
- print("Your output will not be synced to azure because azure-blob-storage is not installed")
-
- def upload_local_file_to_azure(self, local_file_path):
- try:
- from azure.core.exceptions import HttpResponseError
- try:
- blob_client = self._init_blob_client(local_file_path)
- if blob_client:
- with open(local_file_path, "rb") as fin:
- blob_client.upload_blob(fin, overwrite=True)
- except HttpResponseError as err:
- print("Cannot upload blob due to {}: {}".format("azure.core.exceptions.HttpResponseError",
- err))
- except ImportError:
- print("Your output will not be synced to azure because azure-blob-storage is not installed")
-
- def download_azure_blob(self, blobname):
- # TODO coverage
- blob_client = self._init_blob_client(blobname)
- if blob_client:
- pathlib.Path(re.search("(?P<parent_path>^.*)/[^/]+$", blobname).group("parent_path")).mkdir(
- parents=True, exist_ok=True)
- with open(blobname, "wb") as fout:
- fout.write(blob_client.download_blob().readall())
-
- def extract_configscore_list_from_analysis(self,
- analysis):
- """
- Extracting a json object for storing the key information returned from tune.run
- """
- configscore_list = []
- for each_trial in analysis.trials:
- trial_id = each_trial.trial_id
- start_time = each_trial.start_time
- last_update_time = each_trial.last_update_time
- config = each_trial.config
- try:
- metric_score = each_trial.metric_analysis["eval_" + analysis.default_metric]
- time_stamp = each_trial.metric_analysis['timestamp']
- except KeyError:
- # TODO coverage
- print("KeyError, {} does not contain the key {} or {}".format("each_trial.metric_analysis",
- "eval_" + analysis.default_metric,
- "timestamp"))
- metric_score = 0
- time_stamp = 0
- configscore_list.append(ConfigScore(
- trial_id=trial_id,
- start_time=start_time,
- last_update_time=last_update_time,
- config=config,
- metric_score=metric_score,
- time_stamp=time_stamp))
- return configscore_list
-
- def write_autohf_output(self,
- configscore_list=None,
- valid_metric=None,
- predictions=None,
- duration=None):
- """
- Write the key info from a job and upload to azure blob storage
- """
- local_file_path = self.generate_local_json_path()
- output_json = {}
- if configscore_list:
- output_json["val_log"] = [configscore.__dict__ for configscore in configscore_list]
- if valid_metric:
- output_json["valid_metric"] = valid_metric
- if duration:
- output_json["duration"] = duration
- if len(output_json) > 0:
- self.create_local_json_and_upload(output_json, local_file_path)
- if predictions is not None:
- self.create_local_prediction_and_upload(local_file_path, predictions)
-
- def generate_local_json_path(self):
- """
- Return a path string for storing the json file locally
- """
- full_dataset_name = self.jobid.get_jobid_full_data_name()
- jobid_str = self.jobid.to_jobid_string()
- local_file_path = os.path.join(self.root_log_path, full_dataset_name, jobid_str + ".json")
- pathlib.Path(os.path.join(self.root_log_path, full_dataset_name)).mkdir(parents=True, exist_ok=True)
- return local_file_path
-
- def create_local_json_and_upload(self, result_json, local_file_path):
- with open(local_file_path, "w") as fout:
- fout.write(json.dumps(result_json))
- fout.flush()
- self.upload_local_file_to_azure(local_file_path)
-
- def create_local_prediction_and_upload(self,
- local_json_file,
- predictions):
- """
- Store predictions (a .zip file) locally and upload
- """
- azure_save_file_name = local_json_file.split("/")[-1][:-5]
- if self.data_root_dir is None:
- # TODO coverage
- from ..utils import load_dft_args
- console_args = load_dft_args()
- output_dir = getattr(console_args, "data_root_dir")
- print("The path for saving the prediction .zip file is not specified, "
- "setting to {} by default".format(output_dir))
- else:
- output_dir = self.data_root_dir
- local_archive_path = self.autohf.output_prediction(predictions,
- output_prediction_path=output_dir + "result/",
- output_zip_file_name=azure_save_file_name)
- self.upload_local_file_to_azure(local_archive_path)
-
- @staticmethod
- def is_after_earliest_time(this_blob, earliest_time: Tuple[int, int, int]):
- # TODO coverage
- import pytz
- utc = pytz.UTC
- if this_blob.last_modified >= utc.localize(datetime(earliest_time[0], earliest_time[1], earliest_time[2])):
- return True
- return False
-
- def get_configblob_from_partial_jobid(self,
- root_log_path,
- partial_jobid,
- earliest_time: Tuple[int, int, int] = None):
- """
- Get all blobs whose jobid configs match the partial_jobid
- """
- blob_list = []
- container_client = self._init_azure_clients()
- if container_client:
- for each_blob in container_client.list_blobs():
- # TODO coverage
- if each_blob.name.startswith(root_log_path):
- each_jobconfig = JobID.convert_blobname_to_jobid(each_blob.name)
- is_append = False
- if each_jobconfig:
- if each_jobconfig.is_match(partial_jobid):
- is_append = True
- if earliest_time and not AzureUtils.is_after_earliest_time(each_blob, earliest_time):
- is_append = False
- if is_append:
- blob_list.append((each_jobconfig, each_blob))
- return blob_list
-
- def get_config_and_score_from_partial_jobid(self,
- root_log_path: str,
- partial_jobid: JobID,
- earliest_time: Tuple[int, int, int] = None):
- """
- Extract the config and score list from a partial config id
-
- Args:
- root_log_path:
- The root log path in azure blob storage, e.g., "logs_seed/"
-
- partial_jobid:
- The partial jobid for matching the blob list
-
- earliest_time (optional):
- The earliest starting time for any matched blob, for filtering out out-dated jobs,
- format: (YYYY, MM, DD)
-
- Return:
- a ConfigScore list object which stores the config and scores list for each matched blob lists
-
- """
- assert isinstance(root_log_path, str), "root_log_path must be of type str"
- assert isinstance(partial_jobid, JobID), "partial_jobid must be of type JobID"
- if earliest_time:
- assert isinstance(earliest_time, tuple), "earliest_time must be a tuple of (YYYY, MM, DD)"
-
- matched_blob_list = self.get_configblob_from_partial_jobid(
- root_log_path,
- partial_jobid,
- earliest_time=earliest_time)
- return self.get_config_and_score_from_matched_blob_list(matched_blob_list,
- earliest_time)
-
- def get_config_and_score_from_matched_blob_list(self,
- matched_blob_list,
- earliest_time: Tuple[int, int, int] = None):
- """
- Extract the config and score list of one or multiple blobs
-
- Args:
- matched_blob_list:
- matched blob list
-
- Return:
- a ConfigScore list object which stores the config and scores list for each matched blob lists
-
- """
- matched_config_score_lists = []
- for (each_jobconfig, each_blob) in matched_blob_list:
- # TODO coverage
- self.download_azure_blob(each_blob.name)
- data_json = json.load(open(each_blob.name, "r"))
- each_config_and_score_list = ConfigScoreList(
- jobid_config=each_jobconfig,
- blob_file=each_blob,
- config_score_list=[ConfigScore(**each_dict) for each_dict in data_json['val_log']])
- matched_config_score_lists.append(each_config_and_score_list)
- return matched_config_score_lists
diff --git a/flaml/nlp/result_analysis/wandb_utils.py b/flaml/nlp/result_analysis/wandb_utils.py
deleted file mode 100644
index 9c3c4adb9..000000000
--- a/flaml/nlp/result_analysis/wandb_utils.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import os
-import subprocess
-import hashlib
-from time import time
-import json
-
-
-class WandbUtils:
-
- # Documentation on the wandb setting:
- # There are two ways to initialize wandb in tune.run:
- # (1) using WandbLoggerCallback, by adding the following argument to tune.run:
- # callbacks=[WandbLoggerCallback(
- # project="hpo",
- # api_key = os.environ["WANDB_API_KEY"],
- # group = os.environ["WANDB_RUN_GROUP"],
- # log_config=True)]
- # (2) using wandb_mixin decorator (the current implementation)
- # The current implementation uses (2) because (1) has the following bug.
- # In Ray 1.2, when using WandbLoggerCallback + setting time limit using the time_budget_s argument,
- # A bug exists which is the previous run will not clear the cache after tune.run returns. After the
- # later run has already starts, some zombie trials in the previous run remain in the memory and never stop.
- # This bug can be reproduced by switching to (1) by adding the above callbacks argument
- # and removing the wandb_mixin decorator
- # https://docs.ray.io/en/master/tune/tutorials/tune-wandb.html
-
- def __init__(self,
- is_wandb_on=False,
- wandb_key_path=None,
- jobid_config=None):
- if is_wandb_on:
- wandb_key = WandbUtils.get_wandb_key(wandb_key_path)
- if wandb_key != "":
- subprocess.run(["wandb", "login", "--relogin", wandb_key])
- os.environ["WANDB_API_KEY"] = wandb_key
- os.environ["WANDB_MODE"] = "online"
- else:
- # TODO coverage
- os.environ["WANDB_MODE"] = "disabled"
- self.jobid_config = jobid_config
-
- @staticmethod
- def get_wandb_key(key_path):
- try:
- try:
- key_json = json.load(open(os.path.join(key_path, "key.json"), "r"))
- wandb_key = key_json["wandb_key"]
- return wandb_key
- except FileNotFoundError:
- print("Cannot use wandb module because key.json is not found under key_path")
- return ""
- except KeyError:
- print("Cannot use wandb module because wandb key is not specified")
- return ""
-
- def set_wandb_per_trial(self):
- # TODO coverage
- print("before wandb.init\n\n\n")
- try:
- import wandb
- try:
- if os.environ["WANDB_MODE"] == "online":
- os.environ["WANDB_SILENT"] = "false"
- return wandb.init(project=self.jobid_config.get_jobid_full_data_name(),
- group=self.wandb_group_name,
- name=str(WandbUtils._get_next_trial_ids()),
- settings=wandb.Settings(
- _disable_stats=True),
- reinit=False)
- else:
- return None
- except wandb.errors.UsageError as err:
- print(err)
- return None
- except ImportError:
- print("Cannot use wandb module because wandb is not installed, run pip install wandb==0.10.26")
-
- @staticmethod
- def _get_next_trial_ids():
- hash = hashlib.sha1()
- hash.update(str(time()).encode('utf-8'))
- return "trial_" + hash.hexdigest()[:3]
-
- def set_wandb_per_run(self):
- try:
- import wandb
- os.environ["WANDB_RUN_GROUP"] = self.jobid_config.to_wandb_string() + wandb.util.generate_id()
- self.wandb_group_name = os.environ["WANDB_RUN_GROUP"]
- try:
- if os.environ["WANDB_MODE"] == "online":
- os.environ["WANDB_SILENT"] = "false"
- return wandb.init(project=self.jobid_config.get_jobid_full_data_name(),
- group=os.environ["WANDB_RUN_GROUP"],
- settings=wandb.Settings(
- _disable_stats=True),
- reinit=False)
- else:
- return None
- except wandb.errors.UsageError as err:
- print(err)
- return None
- except ImportError:
- print("Cannot use wandb module because wandb is not installed, run pip install wandb==0.10.26")
diff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.py
deleted file mode 100644
index ee4cb40d7..000000000
--- a/flaml/nlp/utils.py
+++ /dev/null
@@ -1,138 +0,0 @@
-import argparse
-import json
-import os
-import pathlib
-import re
-from dataclasses import dataclass, field
-
-
-def dataset_subdataset_name_format_check(val_str):
- regex = re.compile(r"^[^:]*:[^:]*$")
- if (val_str is not None) and (not regex.search(val_str)):
- raise argparse.ArgumentTypeError("dataset_subdataset_name must be in the format {data_name}:{subdata_name}")
- return val_str
-
-
-def pretrained_model_size_format_check(val_str):
- regex = re.compile(r"^[^:]*:(small|base|large|xlarge)")
- if (val_str is not None) and (not regex.search(val_str)):
- raise argparse.ArgumentTypeError("pretrained_model_size must be in the format {model_name}:{model_size},"
- "where {model_name} is the name from huggingface.co/models, {model_size}"
- "is chosen from small, base, large, xlarge")
- return val_str
-
-
-def load_dft_args():
- arg_parser = argparse.ArgumentParser()
- arg_parser.add_argument('--server_name', type=str, help='server name', required=False,
- choices=["tmdev", "dgx", "azureml"], default="tmdev")
- arg_parser.add_argument('--algo_mode', type=str, help='hpo or grid search', required=False,
- choices=["grid", "hpo", "hfhpo"], default="hpo")
- arg_parser.add_argument('--data_root_dir', type=str, help='data dir', required=False, default="data/")
- arg_parser.add_argument('--dataset_subdataset_name', type=dataset_subdataset_name_format_check,
- help='dataset and subdataset name', required=False, default=None)
- arg_parser.add_argument('--space_mode', type=str, help='space mode', required=False,
- choices=["grid", "gnr", "uni", "uni_test", "cus", "buni"], default="uni")
- arg_parser.add_argument('--search_alg_args_mode', type=str, help='search algorithm args mode', required=False,
- choices=["dft", "exp", "cus"], default="dft")
- arg_parser.add_argument('--algo_name', type=str, help='algorithm', required=False,
- choices=["bs", "optuna", "cfo", "rs"], default="bs")
- arg_parser.add_argument('--pruner', type=str, help='pruner', required=False,
- choices=["asha", "None"], default="None")
- arg_parser.add_argument('--pretrained_model_size', type=pretrained_model_size_format_check,
- help='pretrained model', required=False, default=None)
- arg_parser.add_argument('--sample_num', type=int, help='sample num', required=False, default=None)
- arg_parser.add_argument('--time_budget', type=int, help='time budget', required=False, default=None)
- arg_parser.add_argument('--time_as_grid', type=int, help='time as grid search', required=False, default=None)
- arg_parser.add_argument('--rep_id', type=int, help='rep id', required=False, default=0)
- arg_parser.add_argument('--azure_key', type=str, help='azure key', required=False, default=None)
- arg_parser.add_argument('--resplit_mode', type=str, help='resplit mode', required=False,
- choices=["rspt", "ori"], default="ori")
- arg_parser.add_argument('--ds_config', type=str, help='deep speed config file path',
- required=False, default=None)
- arg_parser.add_argument('--yml_file', type=str, help='yml file path', required=False, default="test.yml")
- arg_parser.add_argument('--key_path', type=str, help='path for key.json', required=False, default=None)
- arg_parser.add_argument('--root_log_path', type=str, help='root path for log', required=False, default="logs_azure")
- arg_parser.add_argument('--round_idx', type=int, help='round idx for acl experiments', required=False, default=0)
- arg_parser.add_argument('--seed_data', type=int, help='seed of data shuffling', required=False, default=43)
- arg_parser.add_argument('--seed_transformers', type=int, help='seed of transformers', required=False, default=42)
- console_args, unknown = arg_parser.parse_known_args()
- return console_args
-
-
-def merge_dicts(dict1, dict2):
- for key2 in dict2.keys():
- if key2 in dict1:
- dict1_vals = set(dict1[key2])
- dict2_vals = set(dict2[key2])
- dict1[key2] = list(dict1_vals.union(dict2_vals))
- else:
- dict1[key2] = dict2[key2]
- return dict1
-
-
-def _check_dict_keys_overlaps(dict1: dict, dict2: dict):
- dict1_keys = set(dict1.keys())
- dict2_keys = set(dict2.keys())
- return len(dict1_keys.intersection(dict2_keys)) > 0
-
-
-def _variable_override_default_alternative(obj_ref, var_name, default_value, all_values, overriding_value=None):
- """
- Setting the value of var. If overriding_value is specified, var is set to overriding_value;
- If overriding_value is not specified, var is set to default_value meanwhile showing all_values
- """
- assert isinstance(all_values, list)
- if overriding_value:
- setattr(obj_ref, var_name, overriding_value)
- print("The value for {} is specified as {}".format(var_name, overriding_value))
- else:
- setattr(obj_ref, var_name, default_value)
- print("The value for {} is not specified, setting it to the default value {}. "
- "Alternatively, you can set it to {}".format(var_name, default_value, ",".join(all_values)))
-
-
-@dataclass
-class PathUtils:
- hpo_ckpt_path: str = field(metadata={"help": "the directory for hpo output"})
- hpo_result_path: str = field(metadata={"help": "the directory for hpo result"})
- hpo_log_path: str = field(metadata={"help": "the directory for log"})
- hpo_config_path: str = field(metadata={"help": "the directory for log"})
-
- log_dir_per_run: str = field(metadata={"help": "log directory for each run."})
- result_dir_per_run: str = field(metadata={"help": "result directory for each run."})
- ckpt_dir_per_run: str = field(metadata={"help": "checkpoint directory for each run."})
- ckpt_dir_per_trial: str = field(metadata={"help": "checkpoint directory for each trial."})
-
- def __init__(self,
- jobid_config,
- hpo_data_root_path,
- ):
- self.jobid_config = jobid_config
- self.hpo_data_root_path = hpo_data_root_path
- self.hpo_ckpt_path = os.path.join(hpo_data_root_path, "checkpoint")
- self.hpo_result_path = os.path.join(hpo_data_root_path, "result")
- self.hpo_log_path = self.hpo_result_path
-
- @staticmethod
- def init_and_make_one_dir(dir_path):
- assert dir_path
- if not os.path.exists(dir_path):
- pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
-
- def make_dir_per_run(self):
- jobid_str = self.jobid_config.to_jobid_string()
- self.ckpt_dir_per_run = os.path.join(self.hpo_ckpt_path, jobid_str)
- PathUtils.init_and_make_one_dir(self.ckpt_dir_per_run)
-
- self.result_dir_per_run = os.path.join(self.hpo_result_path, jobid_str)
- PathUtils.init_and_make_one_dir(self.result_dir_per_run)
-
- self.log_dir_per_run = os.path.join(self.hpo_log_path, jobid_str)
- PathUtils.init_and_make_one_dir(self.log_dir_per_run)
-
- def make_dir_per_trial(self, trial_id):
- jobid_str = self.jobid_config.to_jobid_string()
- ckpt_dir_per_run = os.path.join(self.hpo_ckpt_path, jobid_str)
- self.ckpt_dir_per_trial = os.path.join(ckpt_dir_per_run, jobid_str, trial_id)
- PathUtils.init_and_make_one_dir(self.ckpt_dir_per_trial)
diff --git a/notebook/research/acl2021.ipynb b/notebook/research/acl2021.ipynb
deleted file mode 100644
index 13d48846e..000000000
--- a/notebook/research/acl2021.ipynb
+++ /dev/null
@@ -1,809 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Copyright (c) 2020-2021. All rights reserved.\r\n",
- "\r\n",
- "Licensed under the MIT License.\r\n",
- "\r\n",
- "# Troubleshooting HPO for fine-tuning pre-trained language models\r\n",
- "\r\n",
- "## 1. Introduction\r\n",
- "\r\n",
- "In this notebook, we demonstrate a procedure for troubleshooting HPO failure in fine-tuning pre-trained language models (introduced in the following paper):\r\n",
- "\r\n",
- "*[An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models](https://arxiv.org/abs/2106.09204). Xueqing Liu, Chi Wang. ACL-IJCNLP 2021*\r\n",
- "\r\n",
- "Notes:\r\n",
- "\r\n",
- "*In this notebook, we only run each experiment 1 time for simplicity, which is different from the paper (3 times). To reproduce the paper's result, please run 3 repetitions and take the average scores.\r\n",
- "\r\n",
- "*Running this notebook takes about one hour.\r\n",
- "\r\n",
- "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` and `nlp` options:\r\n",
- "\r\n",
- "```bash\r\n",
- "pip install flaml[nlp]\r\n",
- "```\r\n",
- "\r\n",
- "Our paper was developed under transformers version 3.4.0. We uninstall and reinstall transformers==3.4.0:\r\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [],
- "source": [
- "!pip install flaml[nlp]\r\n",
- "!pip install transformers==3.4.0\r\n",
- "from flaml.nlp import AutoTransformers\r\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 2. Initial Experimental Study\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Load dataset \n",
- "\n",
- "Load the dataset using AutoTransformer.prepare_data. In this notebook, we use the Microsoft Research Paraphrasing Corpus (MRPC) dataset and the Electra model as an example:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "console_args has no attribute pretrained_model_size, continue\n",
- "console_args has no attribute dataset_subdataset_name, continue\n",
- "console_args has no attribute algo_mode, continue\n",
- "console_args has no attribute space_mode, continue\n",
- "console_args has no attribute search_alg_args_mode, continue\n",
- "console_args has no attribute algo_name, continue\n",
- "console_args has no attribute pruner, continue\n",
- "console_args has no attribute resplit_mode, continue\n",
- "console_args has no attribute rep_id, continue\n",
- "console_args has no attribute seed_data, continue\n",
- "console_args has no attribute seed_transformers, continue\n",
- "console_args has no attribute learning_rate, continue\n",
- "console_args has no attribute weight_decay, continue\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4)\n",
- "Loading cached processed dataset at /home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4/cache-6a78e5c95406457c.arrow\n",
- "Loading cached processed dataset at /home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4/cache-e8d0f3e04c3b4588.arrow\n",
- "Loading cached processed dataset at /home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4/cache-4b0966b394994163.arrow\n",
- "Loading cached processed dataset at /home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4/cache-6a78e5c95406457c.arrow\n",
- "Loading cached processed dataset at /home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4/cache-e8d0f3e04c3b4588.arrow\n",
- "Loading cached processed dataset at /home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4/cache-4b0966b394994163.arrow\n"
- ]
- }
- ],
- "source": [
- "autohf = AutoTransformers()\n",
- "preparedata_setting = {\n",
- " \"dataset_subdataset_name\": \"glue:mrpc\",\n",
- " \"pretrained_model_size\": \"google/electra-base-discriminator:base\",\n",
- " \"data_root_path\": \"data/\",\n",
- " \"max_seq_length\": 128,\n",
- " }\n",
- "autohf.prepare_data(**preparedata_setting)\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "### Running grid search\n",
- "\n",
- "First, we run grid search using Electra. By specifying `algo_mode=\"grid\"`, AutoTransformers will run the grid search algorithm. By specifying `space_mode=\"grid\"`, AutoTransformers will use the default grid search configuration recommended by the Electra paper:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- },
- "scrolled": true
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "== Status ==<br>Memory usage on this node: 14.2/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/250.73 GiB heap, 0.0/76.9 GiB objects (0/1.0 accelerator_type:V100)<br>Current best trial: 67d99_00002 with accuracy=0.7254901960784313 and parameters={'learning_rate': 0.0001, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'warmup_ratio': 0.1, 'per_device_train_batch_size': 32, 'hidden_dropout_prob': 0.1, 'attention_probs_dropout_prob': 0.1, 'num_train_epochs': 0.5, 'seed': 42}<br>Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/checkpoint/dat=glue_subdat=mrpc_mod=grid_spa=grid_arg=dft_alg=grid_pru=None_pre=electra_presz=base_spt=ori_rep=0_sddt=43_sdhf=42_var1=None_var2=None/ray_result<br>Number of trials: 4/4 (4 TERMINATED)<br><br>"
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2021-06-16 10:45:35,071\tINFO tune.py:450 -- Total run time: 106.56 seconds (106.41 seconds for the tuning loop).\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Total running time: 106.57789206504822 seconds\n"
- ]
- }
- ],
- "source": [
- "import transformers\n",
- "autohf_settings = {\n",
- " \"resources_per_trial\": {\"gpu\": 1, \"cpu\": 1},\n",
- " \"num_samples\": 1,\n",
- " \"time_budget\": 100000, # unlimited time budget\n",
- " \"ckpt_per_epoch\": 5,\n",
- " \"fp16\": True,\n",
- " \"algo_mode\": \"grid\", # set the search algorithm to grid search\n",
- " \"space_mode\": \"grid\", # set the search space to the recommended grid space\n",
- " \"transformers_verbose\": transformers.logging.ERROR\n",
- " }\n",
- "validation_metric, analysis = autohf.fit(**autohf_settings)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Get the time for running grid search: "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "grid search for glue_mrpc took 106.57789206504822 seconds\n"
- ]
- }
- ],
- "source": [
- "GST = autohf.last_run_duration\n",
- "print(\"grid search for {} took {} seconds\".format(autohf.jobid_config.get_jobid_full_data_name(), GST))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "After the HPO run finishes, generate the predictions and save it as a .zip file to be submitted to the glue website. Here we will need the library AzureUtils which is for storing the output information (e.g., analysis log, .zip file) locally and uploading the output to an azure blob container (e.g., if multiple jobs are executed in a cluster). If the azure key and container information is not specified, the output information will only be saved locally. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "remove_columns_ is deprecated and will be removed in the next major version of datasets. Use the dataset.remove_columns method instead.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Cleaning the existing label column from test data\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " <div>\n",
- " <style>\n",
- " /* Turns off some styling */\n",
- " progress {\n",
- " /* gets rid of default border in Firefox and Opera. */\n",
- " border: none;\n",
- " /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
- " background-size: auto;\n",
- " }\n",
- " </style>\n",
- " \n",
- " <progress value='432' max='432' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
- " [432/432 00:34]\n",
- " </div>\n",
- " "
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "JobID(dat=['glue'], subdat='mrpc', mod='grid', spa='grid', arg='dft', alg='grid', pru='None', pre_full='google/electra-base-discriminator', pre='electra', presz='base', spt='ori', rep=0, sddt=43, sdhf=42, var1=None, var2=None)\n",
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "The path for saving the prediction .zip file is not specified, setting to data/ by default\n",
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "{'eval_accuracy': 0.7254901960784313, 'eval_f1': 0.8276923076923076, 'eval_loss': 0.516851007938385}\n"
- ]
- }
- ],
- "source": [
- "predictions, test_metric = autohf.predict()\n",
- "from flaml.nlp import AzureUtils\n",
- "\n",
- "print(autohf.jobid_config)\n",
- "\n",
- "azure_utils = AzureUtils(root_log_path=\"logs_test/\", autohf=autohf)\n",
- "azure_utils.write_autohf_output(valid_metric=validation_metric,\n",
- " predictions=predictions,\n",
- " duration=GST)\n",
- "print(validation_metric)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "The validation F1/accuracy we got was 92.4/89.5. After the above steps, you will find a .zip file for the predictions under data/result/. Submit the .zip file to the glue website. The test F1/accuracy we got was 90.4/86.7. As an example, we only run the experiment one time, but in general, we should run the experiment multiple repetitions and report the averaged validation and test accuracy."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "### Running Random Search\n",
- "\n",
- "Next, we run random search with the same time budget as grid search:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [],
- "source": [
- "def tune_hpo(time_budget, this_hpo_space):\n",
- " autohf_settings = {\n",
- " \"resources_per_trial\": {\"gpu\": 1, \"cpu\": 1},\n",
- " \"num_samples\": -1,\n",
- " \"time_budget\": time_budget,\n",
- " \"ckpt_per_epoch\": 5,\n",
- " \"fp16\": True,\n",
- " \"algo_mode\": \"hpo\", # set the search algorithm mode to hpo\n",
- " \"algo_name\": \"rs\",\n",
- " \"space_mode\": \"cus\", # customized search space (this_hpo_space)\n",
- " \"hpo_space\": this_hpo_space,\n",
- " \"transformers_verbose\": transformers.logging.ERROR\n",
- " }\n",
- " validation_metric, analysis = autohf.fit(**autohf_settings)\n",
- " predictions, test_metric = autohf.predict()\n",
- " azure_utils = AzureUtils(root_log_path=\"logs_test/\", autohf=autohf)\n",
- " azure_utils.write_autohf_output(valid_metric=validation_metric,\n",
- " predictions=predictions,\n",
- " duration=GST)\n",
- " print(validation_metric)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "== Status ==<br>Memory usage on this node: 30.1/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/247.51 GiB heap, 0.0/75.93 GiB objects (0/1.0 accelerator_type:V100)<br>Current best trial: c67b4_00003 with accuracy=0.7303921568627451 and parameters={'learning_rate': 4.030097060410288e-05, 'warmup_ratio': 0.06084844859190755, 'num_train_epochs': 0.5, 'per_device_train_batch_size': 16, 'weight_decay': 0.15742692948967135, 'attention_probs_dropout_prob': 0.08638900372842316, 'hidden_dropout_prob': 0.058245828039608386, 'seed': 42}<br>Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/checkpoint/dat=glue_subdat=mrpc_mod=hpo_spa=cus_arg=dft_alg=rs_pru=None_pre=electra_presz=base_spt=ori_rep=0_sddt=43_sdhf=42_var1=None_var2=None/ray_result<br>Number of trials: 8/infinite (8 TERMINATED)<br><br>"
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[2m\u001b[36m(pid=50964)\u001b[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n",
- "\u001b[2m\u001b[36m(pid=50964)\u001b[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n",
- "\u001b[2m\u001b[36m(pid=50948)\u001b[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n",
- "\u001b[2m\u001b[36m(pid=50948)\u001b[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2021-06-16 10:48:21,624\tINFO tune.py:450 -- Total run time: 114.32 seconds (109.41 seconds for the tuning loop).\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Total running time: 114.35665488243103 seconds\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " <div>\n",
- " <style>\n",
- " /* Turns off some styling */\n",
- " progress {\n",
- " /* gets rid of default border in Firefox and Opera. */\n",
- " border: none;\n",
- " /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
- " background-size: auto;\n",
- " }\n",
- " </style>\n",
- " \n",
- " <progress value='432' max='432' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
- " [432/432 00:33]\n",
- " </div>\n",
- " "
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "The path for saving the prediction .zip file is not specified, setting to data/ by default\n",
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "{'eval_accuracy': 0.7328431372549019, 'eval_f1': 0.8320493066255777, 'eval_loss': 0.5411379933357239}\n"
- ]
- }
- ],
- "source": [
- "hpo_space_full = {\n",
- " \"learning_rate\": {\"l\": 3e-5, \"u\": 1.5e-4, \"space\": \"log\"},\n",
- " \"warmup_ratio\": {\"l\": 0, \"u\": 0.2, \"space\": \"linear\"},\n",
- " \"num_train_epochs\": [3],\n",
- " \"per_device_train_batch_size\": [16, 32, 64],\n",
- " \"weight_decay\": {\"l\": 0.0, \"u\": 0.3, \"space\": \"linear\"},\n",
- " \"attention_probs_dropout_prob\": {\"l\": 0, \"u\": 0.2, \"space\": \"linear\"},\n",
- " \"hidden_dropout_prob\": {\"l\": 0, \"u\": 0.2, \"space\": \"linear\"},\n",
- " }\n",
- "\n",
- "tune_hpo(GST, hpo_space_full)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "The validation F1/accuracy we got was 93.5/90.9. Similarly, we can submit the .zip file to the glue website. The test F1/accuaracy we got was 81.6/70.2. "
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "## 3. Troubleshooting HPO Failures\n",
- "\n",
- "Since the validation accuracy is larger than grid search while the test accuracy is smaller, HPO has overfitting. We reduce the search space:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "== Status ==<br>Memory usage on this node: 26.5/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/247.51 GiB heap, 0.0/75.93 GiB objects (0/1.0 accelerator_type:V100)<br>Current best trial: 234d8_00003 with accuracy=0.7475490196078431 and parameters={'learning_rate': 0.00011454435497690623, 'warmup_ratio': 0.1, 'num_train_epochs': 0.5, 'per_device_train_batch_size': 16, 'weight_decay': 0.06370173320348284, 'attention_probs_dropout_prob': 0.03636499344142013, 'hidden_dropout_prob': 0.03668090197068676, 'seed': 42}<br>Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/checkpoint/dat=glue_subdat=mrpc_mod=hpo_spa=cus_arg=dft_alg=rs_pru=None_pre=electra_presz=base_spt=ori_rep=0_sddt=43_sdhf=42_var1=None_var2=None/ray_result<br>Number of trials: 6/infinite (6 TERMINATED)<br><br>"
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2021-06-16 10:51:34,598\tINFO tune.py:450 -- Total run time: 151.57 seconds (136.77 seconds for the tuning loop).\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Total running time: 151.59901237487793 seconds\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " <div>\n",
- " <style>\n",
- " /* Turns off some styling */\n",
- " progress {\n",
- " /* gets rid of default border in Firefox and Opera. */\n",
- " border: none;\n",
- " /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
- " background-size: auto;\n",
- " }\n",
- " </style>\n",
- " \n",
- " <progress value='432' max='432' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
- " [432/432 00:33]\n",
- " </div>\n",
- " "
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "The path for saving the prediction .zip file is not specified, setting to data/ by default\n",
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "{'eval_accuracy': 0.7475490196078431, 'eval_f1': 0.8325203252032519, 'eval_loss': 0.5056071877479553}\n"
- ]
- }
- ],
- "source": [
- "hpo_space_fixwr = {\n",
- " \"learning_rate\": {\"l\": 3e-5, \"u\": 1.5e-4, \"space\": \"log\"},\n",
- " \"warmup_ratio\": [0.1],\n",
- " \"num_train_epochs\": [3],\n",
- " \"per_device_train_batch_size\": [16, 32, 64],\n",
- " \"weight_decay\": {\"l\": 0.0, \"u\": 0.3, \"space\": \"linear\"},\n",
- " \"attention_probs_dropout_prob\": {\"l\": 0, \"u\": 0.2, \"space\": \"linear\"},\n",
- " \"hidden_dropout_prob\": {\"l\": 0, \"u\": 0.2, \"space\": \"linear\"},\n",
- " }\n",
- "tune_hpo(GST, hpo_space_fixwr)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The validation F1/accuracy we got was 92.6/89.7, the test F1/accuracy was 85.9/78.7, therefore overfitting still exists and we further reduce the space: "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "== Status ==<br>Memory usage on this node: 29.6/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/247.46 GiB heap, 0.0/75.93 GiB objects (0/1.0 accelerator_type:V100)<br>Current best trial: 96a67_00003 with accuracy=0.7107843137254902 and parameters={'learning_rate': 7.862589064613256e-05, 'warmup_ratio': 0.1, 'num_train_epochs': 0.5, 'per_device_train_batch_size': 32, 'weight_decay': 0.0, 'attention_probs_dropout_prob': 0.1, 'hidden_dropout_prob': 0.1, 'seed': 42}<br>Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/checkpoint/dat=glue_subdat=mrpc_mod=hpo_spa=cus_arg=dft_alg=rs_pru=None_pre=electra_presz=base_spt=ori_rep=0_sddt=43_sdhf=42_var1=None_var2=None/ray_result<br>Number of trials: 6/infinite (6 TERMINATED)<br><br>"
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n",
- "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n",
- "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2021-06-16 10:54:14,542\tINFO tune.py:450 -- Total run time: 117.99 seconds (112.99 seconds for the tuning loop).\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Total running time: 118.01927375793457 seconds\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " <div>\n",
- " <style>\n",
- " /* Turns off some styling */\n",
- " progress {\n",
- " /* gets rid of default border in Firefox and Opera. */\n",
- " border: none;\n",
- " /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
- " background-size: auto;\n",
- " }\n",
- " </style>\n",
- " \n",
- " <progress value='432' max='432' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
- " [432/432 00:33]\n",
- " </div>\n",
- " "
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "The path for saving the prediction .zip file is not specified, setting to data/ by default\n",
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "{'eval_accuracy': 0.7181372549019608, 'eval_f1': 0.8174962292609351, 'eval_loss': 0.5494586229324341}\n"
- ]
- }
- ],
- "source": [
- "hpo_space_min = {\n",
- " \"learning_rate\": {\"l\": 3e-5, \"u\": 1.5e-4, \"space\": \"log\"},\n",
- " \"warmup_ratio\": [0.1],\n",
- " \"num_train_epochs\": [3],\n",
- " \"per_device_train_batch_size\": [16, 32, 64],\n",
- " \"weight_decay\": [0.0],\n",
- " \"attention_probs_dropout_prob\": [0.1],\n",
- " \"hidden_dropout_prob\": [0.1],\n",
- " }\n",
- "tune_hpo(GST, hpo_space_min)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "The validation F1/accuracy we got was 90.4/86.7, test F1/accuracy was 83.0/73.0. Since the validation accuracy is below grid search, we increase the budget to 4 * GST:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "== Status ==<br>Memory usage on this node: 26.2/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/247.46 GiB heap, 0.0/75.93 GiB objects (0/1.0 accelerator_type:V100)<br>Current best trial: f5d31_00005 with accuracy=0.7352941176470589 and parameters={'learning_rate': 3.856175093679045e-05, 'warmup_ratio': 0.1, 'num_train_epochs': 0.5, 'per_device_train_batch_size': 16, 'weight_decay': 0.0, 'attention_probs_dropout_prob': 0.1, 'hidden_dropout_prob': 0.1, 'seed': 42}<br>Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/checkpoint/dat=glue_subdat=mrpc_mod=hpo_spa=cus_arg=dft_alg=rs_pru=None_pre=electra_presz=base_spt=ori_rep=0_sddt=43_sdhf=42_var1=None_var2=None/ray_result<br>Number of trials: 16/infinite (16 TERMINATED)<br><br>"
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n",
- "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n",
- "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n",
- "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2021-06-16 11:03:23,308\tINFO tune.py:450 -- Total run time: 507.09 seconds (445.79 seconds for the tuning loop).\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Total running time: 507.15925645828247 seconds\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- " <div>\n",
- " <style>\n",
- " /* Turns off some styling */\n",
- " progress {\n",
- " /* gets rid of default border in Firefox and Opera. */\n",
- " border: none;\n",
- " /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
- " background-size: auto;\n",
- " }\n",
- " </style>\n",
- " \n",
- " <progress value='432' max='432' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
- " [432/432 00:34]\n",
- " </div>\n",
- " "
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "The path for saving the prediction .zip file is not specified, setting to data/ by default\n",
- "Your output will not be synced to azure because azure key and container name are not specified\n",
- "{'eval_accuracy': 0.7401960784313726, 'eval_f1': 0.8333333333333334, 'eval_loss': 0.5303606986999512}\n"
- ]
- }
- ],
- "source": [
- "hpo_space_min = {\n",
- " \"learning_rate\": {\"l\": 3e-5, \"u\": 1.5e-4, \"space\": \"log\"},\n",
- " \"warmup_ratio\": [0.1],\n",
- " \"num_train_epochs\": [3],\n",
- " \"per_device_train_batch_size\": [32],\n",
- " \"weight_decay\": [0.0],\n",
- " \"attention_probs_dropout_prob\": [0.1],\n",
- " \"hidden_dropout_prob\": [0.1],\n",
- " }\n",
- "tune_hpo(4 * GST, hpo_space_min)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The validation F1/accuracy we got was 93.5/91.1, where the accuracy outperforms grid search. The test F1/accuracy was 90.1/86.1. As a result, random search with 4*GST and the minimum space overfits. We stop the troubleshooting process because the search space cannot be further reduced."
- ]
- }
- ],
- "metadata": {
- "interpreter": {
- "hash": "bfcd9a6a9254a5e160761a1fd7a9e444f011592c6770d9f4180dde058a9df5dd"
- },
- "kernelspec": {
- "display_name": "Python 3.7.7 64-bit ('flaml': conda)",
- "name": "python3"
- },
- "language_info": {
- "name": "python",
- "version": ""
- }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 01ff683e4..340d4d9b0 100644
--- a/setup.py
+++ b/setup.py
@@ -52,11 +52,6 @@ setuptools.setup(
"optuna==2.8.0",
"vowpalwabbit",
"openml",
- "transformers==4.4.1",
- "wandb==0.10.26",
- "torch==1.8.1",
- "datasets==1.4.1",
- "azure-storage-blob",
"statsmodels>=0.12.2",
"psutil==5.8.0",
],
@@ -75,13 +70,6 @@ setuptools.setup(
"vw": [
"vowpalwabbit",
],
- "nlp": [
- "ray[tune]>=1.6.0",
- "transformers",
- "datasets==1.4.1",
- "tensorboardX<=2.2",
- "torch",
- ],
"ts_forecast": ["prophet>=1.0.1", "statsmodels>=0.12.2"],
"forecast": ["prophet>=1.0.1", "statsmodels>=0.12.2"],
"benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3"],
diff --git a/test/hf/test_cover_azure.py b/test/hf/test_cover_azure.py
deleted file mode 100644
index eb819a52f..000000000
--- a/test/hf/test_cover_azure.py
+++ /dev/null
@@ -1,127 +0,0 @@
-"""
- test suites for covering azure_utils.py
-"""
-
-
-def get_preparedata_setting(jobid_config):
- preparedata_setting = {
- "server_name": "tmdev",
- "data_root_path": "data/",
- "max_seq_length": 128,
- "jobid_config": jobid_config,
- "resplit_portion": {"source": ["train", "validation"],
- "train": [0, 0.8],
- "validation": [0.8, 0.9],
- "test": [0.9, 1.0]}
- }
- return preparedata_setting
-
-
-def get_console_args():
- from flaml.nlp.utils import load_dft_args
- args = load_dft_args()
- args.dataset_subdataset_name = "glue:mrpc"
- args.algo_mode = "hpo"
- args.space_mode = "uni"
- args.search_alg_args_mode = "dft"
- args.algo_name = "bs"
- args.pruner = "None"
- args.pretrained_model_size = "google/electra-base-discriminator:base"
- args.resplit_mode = "rspt"
- args.rep_id = 0
- args.seed_data = 43
- args.seed_transformers = 42
- return args
-
-
-def test_get_configblob_from_partial_jobid():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp import JobID
- each_blob_name = "dat=glue_subdat=cola_mod=grid_spa=cus_arg=dft_alg=grid" \
- "_pru=None_pre=deberta_presz=large_spt=rspt_rep=0_sddt=43" \
- "_sdhf=42.json"
- partial_jobid = JobID()
- partial_jobid.pre = "deberta"
- partial_jobid.mod = "grid"
- partial_jobid.spa = "cus"
- partial_jobid.presz = "large"
-
- each_jobconfig = JobID.convert_blobname_to_jobid(each_blob_name)
- each_jobconfig.is_match(partial_jobid)
-
- partial_jobid = JobID()
- partial_jobid.pre = "deberta"
- partial_jobid.mod = "hpo"
- partial_jobid.spa = "cus"
- partial_jobid.presz = "large"
- partial_jobid.sddt = None
-
- each_jobconfig = JobID.convert_blobname_to_jobid(each_blob_name)
- each_jobconfig.is_match(partial_jobid)
-
-
-def test_jobid():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp import JobID
- args = get_console_args()
-
- jobid_config = JobID(args)
- jobid_config.to_partial_jobid_string()
- JobID.convert_blobname_to_jobid("test")
- JobID.dataset_list_to_str("glue")
- JobID.get_full_data_name(["glue"], "mrpc")
- JobID._extract_model_type_with_keywords_match("google/electra-base-discriminator:base")
-
- jobid_config.to_wandb_string()
-
-
-def test_azureutils():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp import AzureUtils, JobID
- from flaml.nlp.result_analysis.azure_utils import ConfigScore, ConfigScoreList
- from flaml.nlp import AutoTransformers
-
- args = get_console_args()
- args.key_path = "."
- jobid_config = JobID(args)
- autohf = AutoTransformers()
- autohf.jobid_config = jobid_config
-
- preparedata_setting = get_preparedata_setting(jobid_config)
- autohf.prepare_data(**preparedata_setting)
-
- each_configscore = ConfigScore(trial_id="test", start_time=0.0, last_update_time=0.0,
- config={}, metric_score={"max": 0.0}, time_stamp=0.0)
- configscore_list = ConfigScoreList([each_configscore])
- for each_method in ["unsorted", "sort_time", "sort_accuracy"]:
- configscore_list.sorted(each_method)
- configscore_list.get_best_config()
-
- azureutils = AzureUtils(azure_key_path=args.key_path, data_root_dir=args.data_root_dir, autohf=autohf)
- azureutils.autohf = autohf
- azureutils.root_log_path = "logs_azure/"
-
- azureutils.write_autohf_output(configscore_list=[each_configscore],
- valid_metric={},
- predictions=[],
- duration=0)
-
- azureutils.get_config_and_score_from_partial_jobid(root_log_path="data/", partial_jobid=jobid_config)
-
-
-if __name__ == "__main__":
- test_get_configblob_from_partial_jobid()
- test_jobid()
- test_azureutils()
diff --git a/test/hf/test_cover_other.py b/test/hf/test_cover_other.py
deleted file mode 100644
index b3390752f..000000000
--- a/test/hf/test_cover_other.py
+++ /dev/null
@@ -1,214 +0,0 @@
-"""
- test suites for covering other functions
-"""
-
-from transformers import AutoConfig
-from flaml.nlp.huggingface.trainer import TrainerForAutoTransformers
-
-
-def get_console_args():
- from flaml.nlp.utils import load_dft_args
- args = load_dft_args()
- args.dataset_subdataset_name = "glue:mrpc"
- args.algo_mode = "hpo"
- args.space_mode = "uni"
- args.search_alg_args_mode = "dft"
- args.algo_name = "bs"
- args.pruner = "None"
- args.pretrained_model_size = "google/electra-base-discriminator:base"
- args.resplit_mode = "rspt"
- args.rep_id = 0
- args.seed_data = 43
- args.seed_transformers = 42
- return args
-
-
-def model_init():
- from flaml.nlp import JobID
- jobid_config = JobID()
- jobid_config.set_unittest_config()
- from flaml.nlp import AutoTransformers
- autohf = AutoTransformers()
-
- preparedata_setting = get_preparedata_setting(jobid_config)
- autohf.prepare_data(**preparedata_setting)
- return autohf._load_model()
-
-
-def get_preparedata_setting(jobid_config):
- preparedata_setting = {
- "server_name": "tmdev",
- "data_root_path": "data/",
- "max_seq_length": 128,
- "jobid_config": jobid_config,
- "resplit_portion": {"source": ["train", "validation"],
- "train": [0, 0.8],
- "validation": [0.8, 0.9],
- "test": [0.9, 1.0]}
- }
- return preparedata_setting
-
-
-def test_dataprocess():
- """
- test to increase the coverage for flaml.nlp.dataprocess_auto
- """
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp import AutoTransformers
- from flaml.nlp import JobID
- from flaml.nlp.dataset.dataprocess_auto import TOKENIZER_MAPPING
-
- jobid_config = JobID()
- jobid_config.set_unittest_config()
- autohf = AutoTransformers()
-
- dataset_name = JobID.dataset_list_to_str(jobid_config.dat)
- default_func = TOKENIZER_MAPPING[(dataset_name, jobid_config.subdat)]
-
- funcs_to_eval = set([(dat, subdat) for (dat, subdat) in TOKENIZER_MAPPING.keys()
- if TOKENIZER_MAPPING[(dat, subdat)] != default_func])
-
- for (dat, subdat) in funcs_to_eval:
- print("loading dataset for {}, {}".format(dat, subdat))
- jobid_config.dat = dat.split(",")
- jobid_config.subdat = subdat
-
- preparedata_setting = get_preparedata_setting(jobid_config)
- autohf.prepare_data(**preparedata_setting)
-
-
-def test_gridsearch_space():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp.hpo.grid_searchspace_auto import GRID_SEARCH_SPACE_MAPPING, AutoGridSearchSpace
- from flaml.nlp import JobID
- jobid_config = JobID()
- jobid_config.set_unittest_config()
-
- for each_model_type in GRID_SEARCH_SPACE_MAPPING.keys():
- AutoGridSearchSpace.from_model_and_dataset_name(
- each_model_type,
- "base",
- jobid_config.dat,
- jobid_config.subdat, "hpo")
-
-
-def test_hpo_space():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp.hpo.hpo_searchspace import AutoHPOSearchSpace, HPO_SEARCH_SPACE_MAPPING
- from flaml.nlp import JobID
- jobid_config = JobID()
- jobid_config.set_unittest_config()
-
- for spa in HPO_SEARCH_SPACE_MAPPING.keys():
- jobid_config.spa = spa
- if jobid_config.spa == "cus":
- custom_hpo_args = {"hpo_space": {"learning_rate": [1e-5]}}
- elif jobid_config.spa == "buni":
- best_config = {"learning_rate": 1e-5}
- custom_hpo_args = {"points_to_evaluate": [best_config],
- "bound": {"learning_rate": {"u": best_config["learning_rate"]}}}
- else:
- custom_hpo_args = {}
-
- AutoHPOSearchSpace.from_model_and_dataset_name(jobid_config.spa, jobid_config.pre, jobid_config.presz,
- jobid_config.dat, jobid_config.subdat, **custom_hpo_args)
-
-
-def test_trainer():
- try:
- import ray
- except ImportError:
- return
-
- num_train_epochs = 3
- num_train_examples = 100
- per_device_train_batch_size = 32
- device_count = 1
- max_steps = 1000
- warmup_steps = 100
- warmup_ratio = 0.1
- trainer = TrainerForAutoTransformers(model_init=model_init)
- trainer.convert_num_train_epochs_to_max_steps(num_train_epochs,
- num_train_examples,
- per_device_train_batch_size,
- device_count)
- trainer.convert_max_steps_to_num_train_epochs(max_steps,
- num_train_examples,
- per_device_train_batch_size,
- device_count)
- trainer.convert_warmup_ratio_to_warmup_steps(warmup_ratio,
- max_steps=max_steps,
- num_train_epochs=num_train_epochs,
- num_train_examples=num_train_examples,
- per_device_train_batch_size=per_device_train_batch_size,
- device_count=device_count)
- trainer.convert_warmup_steps_to_warmup_ratio(warmup_steps,
- num_train_epochs,
- num_train_examples,
- per_device_train_batch_size,
- device_count)
-
-
-def test_switch_head():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp.huggingface.switch_head_auto import AutoSeqClassificationHead, MODEL_CLASSIFICATION_HEAD_MAPPING
- from flaml.nlp import JobID
- jobid_config = JobID()
- jobid_config.set_unittest_config()
- checkpoint_path = jobid_config.pre_full
-
- model_config = AutoConfig.from_pretrained(
- checkpoint_path,
- num_labels=AutoConfig.from_pretrained(checkpoint_path).num_labels)
-
- for model in list(MODEL_CLASSIFICATION_HEAD_MAPPING.keys()):
- jobid_config.pre = model
- AutoSeqClassificationHead \
- .from_model_type_and_config(jobid_config.pre,
- model_config)
-
-
-def test_wandb_utils():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp.result_analysis.wandb_utils import WandbUtils
- from flaml.nlp import JobID
- import os
-
- args = get_console_args()
- args.key_path = "."
- jobid_config = JobID(args)
-
- wandb_utils = WandbUtils(is_wandb_on=True, wandb_key_path=args.key_path, jobid_config=jobid_config)
- os.environ["WANDB_MODE"] = "online"
- wandb_utils.wandb_group_name = "test"
- wandb_utils._get_next_trial_ids()
- wandb_utils.set_wandb_per_run()
-
-
-if __name__ == "__main__":
- test_wandb_utils()
- test_dataprocess()
- test_gridsearch_space()
- test_hpo_space()
- test_trainer()
- test_switch_head()
diff --git a/test/hf/test_deberta.py b/test/hf/test_deberta.py
deleted file mode 100644
index 18c392785..000000000
--- a/test/hf/test_deberta.py
+++ /dev/null
@@ -1,256 +0,0 @@
-'''Require: pip install torch transformers datasets flaml[blendsearch,ray]
-'''
-import time
-import numpy as np
-import os
-
-try:
- import ray
- from datasets import (
- load_dataset,
- load_metric,
- )
- from transformers import (
- AutoModelForSequenceClassification,
- AutoTokenizer,
- Trainer,
- TrainingArguments,
- )
- import flaml
- MODEL_CHECKPOINT = "microsoft/deberta-base"
- task_to_keys = {
- "cola": ("sentence", None),
- "mnli": ("premise", "hypothesis"),
- "mrpc": ("sentence1", "sentence2"),
- "qnli": ("question", "sentence"),
- "qqp": ("question1", "question2"),
- "rte": ("sentence1", "sentence2"),
- "sst2": ("sentence", None),
- "stsb": ("sentence1", "sentence2"),
- "wnli": ("sentence1", "sentence2"),
- }
- max_seq_length = 128
- overwrite_cache = False
- pad_to_max_length = True
- padding = "max_length"
-
- TASK = "qnli"
- # HP_METRIC, MODE = "loss", "min"
- HP_METRIC, MODE = "accuracy", "max"
-
- sentence1_key, sentence2_key = task_to_keys[TASK]
- # Define tokenize method
- tokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT, use_fast=True)
-
- def tokenize(examples):
- args = (
- (examples[sentence1_key],) if sentence2_key is None else (
- examples[sentence1_key], examples[sentence2_key])
- )
- return tokenizer(*args, padding=padding, max_length=max_seq_length,
- truncation=True)
-
-except ImportError:
- print("pip install torch transformers datasets flaml[blendsearch,ray]")
-except ValueError as e:
- if 'Connection error' in str(e):
- print(e)
- else:
- raise(e)
-
-import logging
-logger = logging.getLogger(__name__)
-os.makedirs('logs', exist_ok=True)
-logger.addHandler(logging.FileHandler('logs/tune_deberta.log'))
-logger.setLevel(logging.INFO)
-
-
-def train_deberta(config: dict):
-
- # Load dataset and apply tokenizer
- data_raw = load_dataset("glue", TASK)
- data_encoded = data_raw.map(tokenize, batched=True)
- train_dataset, eval_dataset = data_encoded["train"], data_encoded["validation"]
-
- NUM_LABELS = len(train_dataset.features["label"].names)
-
- metric = load_metric("glue", TASK)
-
- def compute_metrics(eval_pred):
- predictions, labels = eval_pred
- predictions = np.argmax(predictions, axis=1)
- return metric.compute(predictions=predictions, references=labels)
-
- model = AutoModelForSequenceClassification.from_pretrained(
- MODEL_CHECKPOINT, num_labels=NUM_LABELS
- )
-
- training_args = TrainingArguments(
- output_dir='.',
- do_eval=False,
- disable_tqdm=True,
- logging_steps=20000,
- save_total_limit=0,
- fp16=True,
- **config,
- )
-
- trainer = Trainer(
- model,
- training_args,
- train_dataset=train_dataset,
- eval_dataset=eval_dataset,
- tokenizer=tokenizer,
- compute_metrics=compute_metrics,
- )
-
- # train model
- trainer.train()
-
- # evaluate model
- eval_output = trainer.evaluate()
-
- flaml.tune.report(
- loss=eval_output["eval_loss"],
- accuracy=eval_output["eval_accuracy"],
- )
-
- try:
- from azureml.core import Run
- run = Run.get_context()
- run.log('accuracy', eval_output["eval_accuracy"])
- run.log('loss', eval_output["eval_loss"])
- run.log('config', config)
- except ImportError:
- pass
-
-
-def _test_deberta(method='BlendSearch'):
-
- max_num_epoch = 100
- num_samples = -1
- time_budget_s = 3600
-
- search_space = {
- # You can mix constants with search space objects.
- "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
- "learning_rate": flaml.tune.loguniform(3e-5, 1.5e-4),
- "weight_decay": flaml.tune.uniform(0, 0.3),
- "per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]),
- "seed": flaml.tune.choice([12, 22, 33, 42]),
- }
-
- start_time = time.time()
- ray.init(num_cpus=4, num_gpus=4)
- if 'ASHA' == method:
- algo = None
- elif 'BOHB' == method:
- from ray.tune.schedulers import HyperBandForBOHB
- from ray.tune.suggest.bohb import tuneBOHB
- algo = tuneBOHB(max_concurrent=4)
- scheduler = HyperBandForBOHB(max_t=max_num_epoch)
- elif 'Optuna' == method:
- from ray.tune.suggest.optuna import OptunaSearch
- algo = OptunaSearch()
- elif 'CFO' == method:
- from flaml import CFO
- algo = CFO(low_cost_partial_config={
- "num_train_epochs": 1,
- "per_device_train_batch_size": 128,
- })
- elif 'BlendSearch' == method:
- from flaml import BlendSearch
- algo = BlendSearch(low_cost_partial_config={
- "num_train_epochs": 1,
- "per_device_train_batch_size": 128,
- })
- elif 'Dragonfly' == method:
- from ray.tune.suggest.dragonfly import DragonflySearch
- algo = DragonflySearch()
- elif 'SkOpt' == method:
- from ray.tune.suggest.skopt import SkOptSearch
- algo = SkOptSearch()
- elif 'Nevergrad' == method:
- from ray.tune.suggest.nevergrad import NevergradSearch
- import nevergrad as ng
- algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
- elif 'ZOOpt' == method:
- from ray.tune.suggest.zoopt import ZOOptSearch
- algo = ZOOptSearch(budget=num_samples)
- elif 'Ax' == method:
- from ray.tune.suggest.ax import AxSearch
- algo = AxSearch(max_concurrent=3)
- elif 'HyperOpt' == method:
- from ray.tune.suggest.hyperopt import HyperOptSearch
- algo = HyperOptSearch()
- scheduler = None
- if method != 'BOHB':
- from ray.tune.schedulers import ASHAScheduler
- scheduler = ASHAScheduler(
- max_t=max_num_epoch,
- grace_period=1)
- scheduler = None
- analysis = ray.tune.run(
- train_deberta,
- metric=HP_METRIC,
- mode=MODE,
- resources_per_trial={"gpu": 4, "cpu": 4},
- config=search_space, local_dir='logs/',
- num_samples=num_samples, time_budget_s=time_budget_s,
- keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC,
- scheduler=scheduler, search_alg=algo)
-
- ray.shutdown()
-
- best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
- metric = best_trial.metric_analysis[HP_METRIC][MODE]
-
- logger.info(f"method={method}")
- logger.info(f"n_trials={len(analysis.trials)}")
- logger.info(f"time={time.time()-start_time}")
- logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
- logger.info(f"Best model parameters: {best_trial.config}")
-
-
-def _test_deberta_cfo():
- _test_deberta('CFO')
-
-
-def _test_deberta_dragonfly():
- _test_deberta('Dragonfly')
-
-
-def _test_deberta_skopt():
- _test_deberta('SkOpt')
-
-
-def _test_deberta_nevergrad():
- _test_deberta('Nevergrad')
-
-
-def _test_deberta_zoopt():
- _test_deberta('ZOOpt')
-
-
-def _test_deberta_ax():
- _test_deberta('Ax')
-
-
-def __test_deberta_hyperopt():
- _test_deberta('HyperOpt')
-
-
-def _test_deberta_optuna():
- _test_deberta('Optuna')
-
-
-def _test_deberta_asha():
- _test_deberta('ASHA')
-
-
-def _test_deberta_bohb():
- _test_deberta('BOHB')
-
-
-if __name__ == "__main__":
- _test_deberta()
diff --git a/test/hf/test_distillbert.py b/test/hf/test_distillbert.py
deleted file mode 100644
index a81e5fece..000000000
--- a/test/hf/test_distillbert.py
+++ /dev/null
@@ -1,225 +0,0 @@
-'''Require: pip install torch transformers datasets flaml[blendsearch,ray]
-'''
-import time
-import numpy as np
-import logging
-import os
-
-try:
- import ray
- from datasets import (
- load_dataset,
- load_metric,
- )
- from transformers import (
- AutoModelForSequenceClassification,
- AutoTokenizer,
- Trainer,
- TrainingArguments,
- )
- import flaml
- MODEL_CHECKPOINT = "distilbert-base-uncased"
- TASK = "cola"
- NUM_LABELS = 2
- COLUMN_NAME = "sentence"
- METRIC_NAME = "matthews_correlation"
-
- # HP_METRIC, MODE = "loss", "min"
- HP_METRIC, MODE = "matthews_correlation", "max"
-
- # Define tokenize method
- tokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT, use_fast=True)
-
- def tokenize(examples):
- return tokenizer(examples[COLUMN_NAME], truncation=True)
-
-except ImportError:
- print("pip install torch transformers datasets flaml[blendsearch,ray]")
-except ValueError as e:
- if 'Connection error' in str(e):
- print(e)
- else:
- raise(e)
-
-logger = logging.getLogger(__name__)
-os.makedirs('logs', exist_ok=True)
-logger.addHandler(logging.FileHandler('logs/tune_distilbert.log'))
-logger.setLevel(logging.INFO)
-
-
-def train_distilbert(config: dict):
-
- metric = load_metric("glue", TASK)
-
- def compute_metrics(eval_pred):
- predictions, labels = eval_pred
- predictions = np.argmax(predictions, axis=1)
- return metric.compute(predictions=predictions, references=labels)
-
- # Load CoLA dataset and apply tokenizer
- cola_raw = load_dataset("glue", TASK)
-
- cola_encoded = cola_raw.map(tokenize, batched=True)
- train_dataset, eval_dataset = cola_encoded["train"], cola_encoded["validation"]
-
- model = AutoModelForSequenceClassification.from_pretrained(
- MODEL_CHECKPOINT, num_labels=NUM_LABELS
- )
-
- training_args = TrainingArguments(
- output_dir='.',
- do_eval=False,
- disable_tqdm=True,
- logging_steps=20000,
- save_total_limit=0,
- **config,
- )
-
- trainer = Trainer(
- model,
- training_args,
- train_dataset=train_dataset,
- eval_dataset=eval_dataset,
- tokenizer=tokenizer,
- compute_metrics=compute_metrics,
- )
-
- # train model
- trainer.train()
-
- # evaluate model
- eval_output = trainer.evaluate()
-
- flaml.tune.report(
- loss=eval_output["eval_loss"],
- matthews_correlation=eval_output["eval_matthews_correlation"],
- )
-
-
-def _test_distillbert(method='BlendSearch'):
-
- max_num_epoch = 64
- num_samples = -1
- time_budget_s = 3600
-
- search_space = {
- # You can mix constants with search space objects.
- "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
- "learning_rate": flaml.tune.loguniform(1e-6, 1e-4),
- "adam_beta1": flaml.tune.uniform(0.8, 0.99),
- "adam_beta2": flaml.tune.loguniform(98e-2, 9999e-4),
- "adam_epsilon": flaml.tune.loguniform(1e-9, 1e-7),
- }
-
- start_time = time.time()
- ray.init(num_cpus=4, num_gpus=4)
- if 'ASHA' == method:
- algo = None
- elif 'BOHB' == method:
- from ray.tune.schedulers import HyperBandForBOHB
- from ray.tune.suggest.bohb import tuneBOHB
- algo = tuneBOHB(max_concurrent=4)
- scheduler = HyperBandForBOHB(max_t=max_num_epoch)
- elif 'Optuna' == method:
- from ray.tune.suggest.optuna import OptunaSearch
- algo = OptunaSearch()
- elif 'CFO' == method:
- from flaml import CFO
- algo = CFO(low_cost_partial_config={
- "num_train_epochs": 1,
- })
- elif 'BlendSearch' == method:
- from flaml import BlendSearch
- algo = BlendSearch(low_cost_partial_config={
- "num_train_epochs": 1,
- })
- elif 'Dragonfly' == method:
- from ray.tune.suggest.dragonfly import DragonflySearch
- algo = DragonflySearch()
- elif 'SkOpt' == method:
- from ray.tune.suggest.skopt import SkOptSearch
- algo = SkOptSearch()
- elif 'Nevergrad' == method:
- from ray.tune.suggest.nevergrad import NevergradSearch
- import nevergrad as ng
- algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
- elif 'ZOOpt' == method:
- from ray.tune.suggest.zoopt import ZOOptSearch
- algo = ZOOptSearch(budget=num_samples)
- elif 'Ax' == method:
- from ray.tune.suggest.ax import AxSearch
- algo = AxSearch(max_concurrent=3)
- elif 'HyperOpt' == method:
- from ray.tune.suggest.hyperopt import HyperOptSearch
- algo = HyperOptSearch()
- scheduler = None
- if method != 'BOHB':
- from ray.tune.schedulers import ASHAScheduler
- scheduler = ASHAScheduler(
- max_t=max_num_epoch,
- grace_period=1)
- scheduler = None
- analysis = ray.tune.run(
- train_distilbert,
- metric=HP_METRIC,
- mode=MODE,
- resources_per_trial={"gpu": 4, "cpu": 4},
- config=search_space, local_dir='logs/',
- num_samples=num_samples, time_budget_s=time_budget_s,
- keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC,
- scheduler=scheduler, search_alg=algo)
-
- ray.shutdown()
-
- best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
- metric = best_trial.metric_analysis[HP_METRIC][MODE]
-
- logger.info(f"method={method}")
- logger.info(f"n_trials={len(analysis.trials)}")
- logger.info(f"time={time.time()-start_time}")
- logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
- logger.info(f"Best model parameters: {best_trial.config}")
-
-
-def _test_distillbert_cfo():
- _test_distillbert('CFO')
-
-
-def _test_distillbert_dragonfly():
- _test_distillbert('Dragonfly')
-
-
-def _test_distillbert_skopt():
- _test_distillbert('SkOpt')
-
-
-def _test_distillbert_nevergrad():
- _test_distillbert('Nevergrad')
-
-
-def _test_distillbert_zoopt():
- _test_distillbert('ZOOpt')
-
-
-def _test_distillbert_ax():
- _test_distillbert('Ax')
-
-
-def __test_distillbert_hyperopt():
- _test_distillbert('HyperOpt')
-
-
-def _test_distillbert_optuna():
- _test_distillbert('Optuna')
-
-
-def _test_distillbert_asha():
- _test_distillbert('ASHA')
-
-
-def _test_distillbert_bohb():
- _test_distillbert('BOHB')
-
-
-if __name__ == "__main__":
- _test_distillbert()
diff --git a/test/hf/test_electra.py b/test/hf/test_electra.py
deleted file mode 100644
index b85e18c09..000000000
--- a/test/hf/test_electra.py
+++ /dev/null
@@ -1,256 +0,0 @@
-'''Require: pip install torch transformers datasets flaml[blendsearch,ray]
-'''
-import time
-import numpy as np
-import os
-
-try:
- import ray
- from datasets import (
- load_dataset,
- load_metric,
- )
- from transformers import (
- AutoModelForSequenceClassification,
- AutoTokenizer,
- Trainer,
- TrainingArguments,
- )
- import flaml
- MODEL_CHECKPOINT = "google/electra-base-discriminator"
- task_to_keys = {
- "cola": ("sentence", None),
- "mnli": ("premise", "hypothesis"),
- "mrpc": ("sentence1", "sentence2"),
- "qnli": ("question", "sentence"),
- "qqp": ("question1", "question2"),
- "rte": ("sentence1", "sentence2"),
- "sst2": ("sentence", None),
- "stsb": ("sentence1", "sentence2"),
- "wnli": ("sentence1", "sentence2"),
- }
- max_seq_length = 128
- overwrite_cache = False
- pad_to_max_length = True
- padding = "max_length"
-
- TASK = "qnli"
- # HP_METRIC, MODE = "loss", "min"
- HP_METRIC, MODE = "accuracy", "max"
-
- sentence1_key, sentence2_key = task_to_keys[TASK]
- # Define tokenize method
- tokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT, use_fast=True)
-
- def tokenize(examples):
- args = (
- (examples[sentence1_key],) if sentence2_key is None else (
- examples[sentence1_key], examples[sentence2_key])
- )
- return tokenizer(*args, padding=padding, max_length=max_seq_length,
- truncation=True)
-
-except ImportError:
- print("pip install torch transformers datasets flaml[blendsearch,ray]")
-except ValueError as e:
- if 'Connection error' in str(e):
- print(e)
- else:
- raise(e)
-
-import logging
-logger = logging.getLogger(__name__)
-os.makedirs('logs', exist_ok=True)
-logger.addHandler(logging.FileHandler('logs/tune_electra.log'))
-logger.setLevel(logging.INFO)
-
-
-def train_electra(config: dict):
-
- # Load dataset and apply tokenizer
- data_raw = load_dataset("glue", TASK)
- data_encoded = data_raw.map(tokenize, batched=True)
- train_dataset, eval_dataset = data_encoded["train"], data_encoded["validation"]
-
- NUM_LABELS = len(train_dataset.features["label"].names)
-
- metric = load_metric("glue", TASK)
-
- def compute_metrics(eval_pred):
- predictions, labels = eval_pred
- predictions = np.argmax(predictions, axis=1)
- return metric.compute(predictions=predictions, references=labels)
-
- model = AutoModelForSequenceClassification.from_pretrained(
- MODEL_CHECKPOINT, num_labels=NUM_LABELS
- )
-
- training_args = TrainingArguments(
- output_dir='.',
- do_eval=False,
- disable_tqdm=True,
- logging_steps=20000,
- save_total_limit=0,
- fp16=True,
- **config,
- )
-
- trainer = Trainer(
- model,
- training_args,
- train_dataset=train_dataset,
- eval_dataset=eval_dataset,
- tokenizer=tokenizer,
- compute_metrics=compute_metrics,
- )
-
- # train model
- trainer.train()
-
- # evaluate model
- eval_output = trainer.evaluate()
-
- flaml.tune.report(
- loss=eval_output["eval_loss"],
- accuracy=eval_output["eval_accuracy"],
- )
-
- try:
- from azureml.core import Run
- run = Run.get_context()
- run.log('accuracy', eval_output["eval_accuracy"])
- run.log('loss', eval_output["eval_loss"])
- run.log('config', config)
- except ImportError:
- pass
-
-
-def _test_electra(method='BlendSearch'):
-
- max_num_epoch = 9
- num_samples = -1
- time_budget_s = 3600
-
- search_space = {
- # You can mix constants with search space objects.
- "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
- "learning_rate": flaml.tune.loguniform(3e-5, 1.5e-4),
- "weight_decay": flaml.tune.uniform(0, 0.3),
- "per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]),
- "seed": flaml.tune.choice([12, 22, 33, 42]),
- }
-
- start_time = time.time()
- ray.init(num_cpus=4, num_gpus=4)
- if 'ASHA' == method:
- algo = None
- elif 'BOHB' == method:
- from ray.tune.schedulers import HyperBandForBOHB
- from ray.tune.suggest.bohb import tuneBOHB
- algo = tuneBOHB(max_concurrent=4)
- scheduler = HyperBandForBOHB(max_t=max_num_epoch)
- elif 'Optuna' == method:
- from ray.tune.suggest.optuna import OptunaSearch
- algo = OptunaSearch()
- elif 'CFO' == method:
- from flaml import CFO
- algo = CFO(low_cost_partial_config={
- "num_train_epochs": 1,
- "per_device_train_batch_size": 128,
- })
- elif 'BlendSearch' == method:
- from flaml import BlendSearch
- algo = BlendSearch(low_cost_partial_config={
- "num_train_epochs": 1,
- "per_device_train_batch_size": 128,
- })
- elif 'Dragonfly' == method:
- from ray.tune.suggest.dragonfly import DragonflySearch
- algo = DragonflySearch()
- elif 'SkOpt' == method:
- from ray.tune.suggest.skopt import SkOptSearch
- algo = SkOptSearch()
- elif 'Nevergrad' == method:
- from ray.tune.suggest.nevergrad import NevergradSearch
- import nevergrad as ng
- algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
- elif 'ZOOpt' == method:
- from ray.tune.suggest.zoopt import ZOOptSearch
- algo = ZOOptSearch(budget=num_samples)
- elif 'Ax' == method:
- from ray.tune.suggest.ax import AxSearch
- algo = AxSearch(max_concurrent=3)
- elif 'HyperOpt' == method:
- from ray.tune.suggest.hyperopt import HyperOptSearch
- algo = HyperOptSearch()
- scheduler = None
- if method != 'BOHB':
- from ray.tune.schedulers import ASHAScheduler
- scheduler = ASHAScheduler(
- max_t=max_num_epoch,
- grace_period=1)
- scheduler = None
- analysis = ray.tune.run(
- train_electra,
- metric=HP_METRIC,
- mode=MODE,
- resources_per_trial={"gpu": 4, "cpu": 4},
- config=search_space, local_dir='logs/',
- num_samples=num_samples, time_budget_s=time_budget_s,
- keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC,
- scheduler=scheduler, search_alg=algo)
-
- ray.shutdown()
-
- best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
- metric = best_trial.metric_analysis[HP_METRIC][MODE]
-
- logger.info(f"method={method}")
- logger.info(f"n_trials={len(analysis.trials)}")
- logger.info(f"time={time.time()-start_time}")
- logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
- logger.info(f"Best model parameters: {best_trial.config}")
-
-
-def _test_electra_cfo():
- _test_electra('CFO')
-
-
-def _test_electra_dragonfly():
- _test_electra('Dragonfly')
-
-
-def _test_electra_skopt():
- _test_electra('SkOpt')
-
-
-def _test_electra_nevergrad():
- _test_electra('Nevergrad')
-
-
-def _test_electra_zoopt():
- _test_electra('ZOOpt')
-
-
-def _test_electra_ax():
- _test_electra('Ax')
-
-
-def __test_electra_hyperopt():
- _test_electra('HyperOpt')
-
-
-def _test_electra_optuna():
- _test_electra('Optuna')
-
-
-def _test_electra_asha():
- _test_electra('ASHA')
-
-
-def _test_electra_bohb():
- _test_electra('BOHB')
-
-
-if __name__ == "__main__":
- _test_electra()
diff --git a/test/hf/test_mobilebert.py b/test/hf/test_mobilebert.py
deleted file mode 100644
index 7c5f90d5b..000000000
--- a/test/hf/test_mobilebert.py
+++ /dev/null
@@ -1,74 +0,0 @@
-'''Require: pip install torch transformers datasets wandb flaml[blendsearch,ray]
-'''
-global azure_log_path
-global azure_key
-
-
-def get_preparedata_setting(jobid_config):
- preparedata_setting = {
- "server_name": "tmdev",
- "data_root_path": "data/",
- "max_seq_length": 128,
- "jobid_config": jobid_config,
- "resplit_portion": {"source": ["train", "validation"],
- "train": [0, 0.8],
- "validation": [0.8, 0.9],
- "test": [0.9, 1.0]}
- }
- return preparedata_setting
-
-
-def get_autohf_settings():
- autohf_settings = {"resources_per_trial": {"cpu": 1},
- "num_samples": 1,
- "time_budget": 100000,
- "ckpt_per_epoch": 1,
- "fp16": False,
- }
- return autohf_settings
-
-
-def test_hpo():
- try:
- import ray
- except ImportError:
- return
-
- from flaml.nlp import AutoTransformers
- from flaml.nlp import JobID
- from flaml.nlp import AzureUtils
-
- jobid_config = JobID()
- jobid_config.set_unittest_config()
- autohf = AutoTransformers()
-
- preparedata_setting = get_preparedata_setting(jobid_config)
- autohf.prepare_data(**preparedata_setting)
-
- autohf_settings = get_autohf_settings()
- autohf_settings["points_to_evaluate"] = [{"learning_rate": 2e-5}]
- validation_metric, analysis = autohf.fit(**autohf_settings)
-
- predictions, test_metric = autohf.predict()
- if test_metric:
- validation_metric.update({"test": test_metric})
-
- azure_utils = AzureUtils(root_log_path="logs_test/", data_root_dir="data/", autohf=autohf)
- azure_utils._azure_key = "test"
- azure_utils._container_name = "test"
-
- configscore_list = azure_utils.extract_configscore_list_from_analysis(analysis)
- azure_utils.write_autohf_output(configscore_list=configscore_list,
- valid_metric=validation_metric,
- predictions=predictions,
- duration=autohf.last_run_duration)
-
- jobid_config.mod = "grid"
- autohf = AutoTransformers()
-
- preparedata_setting = get_preparedata_setting(jobid_config)
- autohf.prepare_data(**preparedata_setting)
-
-
-if __name__ == "__main__":
- test_hpo()
diff --git a/test/hf/test_roberta.py b/test/hf/test_roberta.py
deleted file mode 100644
index b02d03fb4..000000000
--- a/test/hf/test_roberta.py
+++ /dev/null
@@ -1,251 +0,0 @@
-'''Require: pip install torch transformers datasets flaml[blendsearch,ray]
-'''
-import time
-import numpy as np
-import os
-
-try:
- import ray
- from datasets import (
- load_dataset,
- load_metric,
- )
- from transformers import (
- AutoModelForSequenceClassification,
- AutoTokenizer,
- Trainer,
- TrainingArguments,
- )
- import flaml
- MODEL_CHECKPOINT = "roberta-base"
- task_to_keys = {
- "cola": ("sentence", None),
- "mnli": ("premise", "hypothesis"),
- "mrpc": ("sentence1", "sentence2"),
- "qnli": ("question", "sentence"),
- "qqp": ("question1", "question2"),
- "rte": ("sentence1", "sentence2"),
- "sst2": ("sentence", None),
- "stsb": ("sentence1", "sentence2"),
- "wnli": ("sentence1", "sentence2"),
- }
- max_seq_length = 128
- overwrite_cache = False
- pad_to_max_length = True
- padding = "max_length"
-
- TASK = "qnli"
- # HP_METRIC, MODE = "loss", "min"
- HP_METRIC, MODE = "accuracy", "max"
-
- sentence1_key, sentence2_key = task_to_keys[TASK]
- # Define tokenize method
- tokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT, use_fast=True)
-
- def tokenize(examples):
- args = (
- (examples[sentence1_key],) if sentence2_key is None else (
- examples[sentence1_key], examples[sentence2_key])
- )
- return tokenizer(*args, padding=padding, max_length=max_seq_length,
- truncation=True)
-
-except ImportError:
- print("pip install torch transformers datasets flaml[blendsearch,ray]")
-
-import logging
-logger = logging.getLogger(__name__)
-os.makedirs('logs', exist_ok=True)
-logger.addHandler(logging.FileHandler('logs/tune_roberta.log'))
-logger.setLevel(logging.INFO)
-
-
-def train_roberta(config: dict):
-
- # Load dataset and apply tokenizer
- data_raw = load_dataset("glue", TASK)
- data_encoded = data_raw.map(tokenize, batched=True)
- train_dataset, eval_dataset = data_encoded["train"], data_encoded["validation"]
-
- NUM_LABELS = len(train_dataset.features["label"].names)
-
- metric = load_metric("glue", TASK)
-
- def compute_metrics(eval_pred):
- predictions, labels = eval_pred
- predictions = np.argmax(predictions, axis=1)
- return metric.compute(predictions=predictions, references=labels)
-
- model = AutoModelForSequenceClassification.from_pretrained(
- MODEL_CHECKPOINT, num_labels=NUM_LABELS
- )
-
- training_args = TrainingArguments(
- output_dir='.',
- do_eval=False,
- disable_tqdm=True,
- logging_steps=20000,
- save_total_limit=0,
- fp16=True,
- **config,
- )
-
- trainer = Trainer(
- model,
- training_args,
- train_dataset=train_dataset,
- eval_dataset=eval_dataset,
- tokenizer=tokenizer,
- compute_metrics=compute_metrics,
- )
-
- # train model
- trainer.train()
-
- # evaluate model
- eval_output = trainer.evaluate()
-
- flaml.tune.report(
- loss=eval_output["eval_loss"],
- accuracy=eval_output["eval_accuracy"],
- )
-
- try:
- from azureml.core import Run
- run = Run.get_context()
- run.log('accuracy', eval_output["eval_accuracy"])
- run.log('loss', eval_output["eval_loss"])
- run.log('config', config)
- except ImportError:
- pass
-
-
-def _test_roberta(method='BlendSearch'):
-
- max_num_epoch = 100
- num_samples = -1
- time_budget_s = 3600
-
- search_space = {
- # You can mix constants with search space objects.
- "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
- "learning_rate": flaml.tune.loguniform(1e-5, 3e-5),
- "weight_decay": flaml.tune.uniform(0, 0.3),
- "per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]),
- "seed": flaml.tune.choice([12, 22, 33, 42]),
- }
-
- start_time = time.time()
- ray.init(num_cpus=4, num_gpus=4)
- if 'ASHA' == method:
- algo = None
- elif 'BOHB' == method:
- from ray.tune.schedulers import HyperBandForBOHB
- from ray.tune.suggest.bohb import tuneBOHB
- algo = tuneBOHB(max_concurrent=4)
- scheduler = HyperBandForBOHB(max_t=max_num_epoch)
- elif 'Optuna' == method:
- from ray.tune.suggest.optuna import OptunaSearch
- algo = OptunaSearch()
- elif 'CFO' == method:
- from flaml import CFO
- algo = CFO(low_cost_partial_config={
- "num_train_epochs": 1,
- "per_device_train_batch_size": 128,
- })
- elif 'BlendSearch' == method:
- from flaml import BlendSearch
- algo = BlendSearch(low_cost_partial_config={
- "num_train_epochs": 1,
- "per_device_train_batch_size": 128,
- })
- elif 'Dragonfly' == method:
- from ray.tune.suggest.dragonfly import DragonflySearch
- algo = DragonflySearch()
- elif 'SkOpt' == method:
- from ray.tune.suggest.skopt import SkOptSearch
- algo = SkOptSearch()
- elif 'Nevergrad' == method:
- from ray.tune.suggest.nevergrad import NevergradSearch
- import nevergrad as ng
- algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
- elif 'ZOOpt' == method:
- from ray.tune.suggest.zoopt import ZOOptSearch
- algo = ZOOptSearch(budget=num_samples)
- elif 'Ax' == method:
- from ray.tune.suggest.ax import AxSearch
- algo = AxSearch(max_concurrent=3)
- elif 'HyperOpt' == method:
- from ray.tune.suggest.hyperopt import HyperOptSearch
- algo = HyperOptSearch()
- scheduler = None
- if method != 'BOHB':
- from ray.tune.schedulers import ASHAScheduler
- scheduler = ASHAScheduler(
- max_t=max_num_epoch,
- grace_period=1)
- scheduler = None
- analysis = ray.tune.run(
- train_roberta,
- metric=HP_METRIC,
- mode=MODE,
- resources_per_trial={"gpu": 4, "cpu": 4},
- config=search_space, local_dir='logs/',
- num_samples=num_samples, time_budget_s=time_budget_s,
- keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC,
- scheduler=scheduler, search_alg=algo)
-
- ray.shutdown()
-
- best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
- metric = best_trial.metric_analysis[HP_METRIC][MODE]
-
- logger.info(f"method={method}")
- logger.info(f"n_trials={len(analysis.trials)}")
- logger.info(f"time={time.time()-start_time}")
- logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
- logger.info(f"Best model parameters: {best_trial.config}")
-
-
-def _test_roberta_cfo():
- _test_roberta('CFO')
-
-
-def _test_roberta_dragonfly():
- _test_roberta('Dragonfly')
-
-
-def _test_roberta_skopt():
- _test_roberta('SkOpt')
-
-
-def _test_roberta_nevergrad():
- _test_roberta('Nevergrad')
-
-
-def _test_roberta_zoopt():
- _test_roberta('ZOOpt')
-
-
-def _test_roberta_ax():
- _test_roberta('Ax')
-
-
-def __test_roberta_hyperopt():
- _test_roberta('HyperOpt')
-
-
-def _test_roberta_optuna():
- _test_roberta('Optuna')
-
-
-def _test_roberta_asha():
- _test_roberta('ASHA')
-
-
-def _test_roberta_bohb():
- _test_roberta('BOHB')
-
-
-if __name__ == "__main__":
- _test_roberta()
diff --git a/test/test_training_log.py b/test/test_training_log.py
index de1be2f35..3c32310a0 100644
--- a/test/test_training_log.py
+++ b/test/test_training_log.py
@@ -67,6 +67,16 @@ class TestTrainingLog(unittest.TestCase):
and str(model.estimator.get_all_params())
== str(automl.model.estimator.get_all_params())
)
+ automl.fit(
+ X_train=X_train,
+ y_train=y_train,
+ max_iter=1,
+ task="regression",
+ estimator_list=[estimator],
+ n_jobs=1,
+ starting_points={estimator: {}},
+ )
+ print(automl.best_config)
with training_log_reader(filename) as reader:
count = 0