A programming framework for agentic AI
fixing bug for ner (#463)
Xueqing Liuaf423463c3d0
diff --git a/flaml/automl.py b/flaml/automl.pyindex cde608a94..1571ece49 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -246,6 +246,7 @@ class AutoMLState:* sample_size/ state.data_size[0])+ # raise Exception("bbbbb", state.time_budget, budget)if _is_nlp_task(state.task):state.fit_kwargs["X_val"] = state.X_val@@ -326,80 +327,29 @@ class AutoMLState:weight = Noneif groups is not None:self.fit_kwargs["groups"] = groups+budget = (Noneif self.time_budget is Noneelse self.time_budget - self.time_from_start)- if (- hasattr(self, "resources_per_trial")- and self.resources_per_trial.get("gpu", 0) > 0- ):-- if _is_nlp_task(self.task):- use_ray = self.fit_kwargs.get("use_ray")- self.fit_kwargs["use_ray"] = True-- def _trainable_function_wrapper(config: dict):-- return_estimator, train_time = train_estimator(- X_train=sampled_X_train,- y_train=sampled_y_train,- config_dic=config,- task=self.task,- estimator_name=estimator,- n_jobs=self.n_jobs,- estimator_class=self.learner_classes.get(estimator),- budget=budget,- fit_kwargs=self.fit_kwargs,- )- return {"estimator": return_estimator, "train_time": train_time}-- if estimator not in self.learner_classes:- self.learner_classes[estimator] = get_estimator_class(- self.task, estimator- )- analysis = tune.run(- _trainable_function_wrapper,- config=config_w_resource,- metric="train_time",- mode="min",- resources_per_trial=self.resources_per_trial,- num_samples=1,- use_ray=True,- )- result = list(analysis.results.values())[0]- estimator, train_time = result["estimator"], result["train_time"]+ estimator, train_time = train_estimator(+ X_train=sampled_X_train,+ y_train=sampled_y_train,+ config_dic=config,+ task=self.task,+ estimator_name=estimator,+ n_jobs=self.n_jobs,+ estimator_class=self.learner_classes.get(estimator),+ budget=budget,+ fit_kwargs=self.fit_kwargs,+ eval_metric="train_time",+ )- if _is_nlp_task(self.task):- if use_ray is None:- del self.fit_kwargs["use_ray"]- else:- self.fit_kwargs["use_ray"] = use_ray- estimator.use_ray = False- else:- if _is_nlp_task(self.task):- use_ray = self.fit_kwargs.get("use_ray")- self.fit_kwargs["use_ray"] = False- estimator, train_time = train_estimator(- X_train=sampled_X_train,- y_train=sampled_y_train,- config_dic=config,- task=self.task,- estimator_name=estimator,- n_jobs=self.n_jobs,- estimator_class=self.learner_classes.get(estimator),- budget=budget,- fit_kwargs=self.fit_kwargs,- )- if _is_nlp_task(self.task):- if use_ray is None:- del self.fit_kwargs["use_ray"]- else:- self.fit_kwargs["use_ray"] = use_rayif sampled_weight is not None:self.fit_kwargs["sample_weight"] = weight+return estimator, train_time@@ -749,7 +699,11 @@ class AutoML(BaseEstimator):"""Time taken to find best model in seconds."""return self.__dict__.get("_time_taken_best_iter")- def predict(self, X: Union[np.array, pd.DataFrame, List[str], List[List[str]]]):+ def predict(+ self,+ X: Union[np.array, pd.DataFrame, List[str], List[List[str]]],+ **pred_kwargs,+ ):"""Predict label from features.Args:@@ -761,6 +715,8 @@ class AutoML(BaseEstimator):arima or sarimax). Other columns in the dataframeare assumed to be exogenous variables (categoricalor numeric).+ **pred_kwargs: Other key word arguments to pass to predict() function of+ the searched learners, such as per_device_eval_batch_size.```pythonmultivariate_X_test = pd.DataFrame({@@ -782,7 +738,7 @@ class AutoML(BaseEstimator):)return NoneX = self._preprocess(X)- y_pred = estimator.predict(X)+ y_pred = estimator.predict(X, **pred_kwargs)if (isinstance(y_pred, np.ndarray)and y_pred.ndim > 1@@ -796,12 +752,14 @@ class AutoML(BaseEstimator):else:return y_pred- def predict_proba(self, X):+ def predict_proba(self, X, **pred_kwargs):"""Predict the probability of each class from features, only works forclassification problems.Args:X: A numpy array of featurized instances, shape n * m.+ **pred_kwargs: Other key word arguments to pass to predict_proba() function of+ the searched learners, such as per_device_eval_batch_size.Returns:A numpy array of shape n * c. c is the # classes. Each element at@@ -814,7 +772,7 @@ class AutoML(BaseEstimator):)return NoneX = self._preprocess(X)- proba = self._trained_estimator.predict_proba(X)+ proba = self._trained_estimator.predict_proba(X, **pred_kwargs)return probadef _preprocess(self, X):@@ -1319,6 +1277,7 @@ class AutoML(BaseEstimator):task=task,estimator_name=estimator,estimator_class=self._state.learner_classes.get(estimator),+ eval_metric="train_time",)return estimator@@ -1680,6 +1639,17 @@ class AutoML(BaseEstimator):"""return self._state.data_size[0] if self._sample else None+ def pickle(self, output_file_name):+ import pickle++ estimator_to_training_function = {}+ for estimator in self.estimator_list:+ search_state = self._search_states[estimator]+ estimator_to_training_function[estimator] = search_state.training_function+ del search_state.training_function+ with open(output_file_name, "wb") as f:+ pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)+@propertydef trainable(self) -> Callable[[dict], Optional[float]]:"""Training function.@@ -1960,10 +1930,10 @@ class AutoML(BaseEstimator):augment rare classes.min_sample_size: int, default=MIN_SAMPLE_TRAIN | the minimal samplesize when sample=True.- use_ray: boolean, default=False | Whether to use ray to run the training+ use_ray: boolean or dict+ If boolean: default=False | Whether to use ray to run the trainingin separate processes. This can be used to prevent OOM for large- datasets, but will incur more overhead in time. Only use it if- you run into OOM failures.+ datasets, but will incur more overhead in time.metric_constraints: list, default=[] | The list of metric constraints.Each element in this list is a 3-tuple, which shall be expressedin the following format: the first element of the 3-tuple is the name of the@@ -2064,14 +2034,21 @@ class AutoML(BaseEstimator):import rayn_cpus = use_ray and ray.available_resources()["CPU"] or os.cpu_count()+self._state.resources_per_trial = (# when using gpu, default cpu is 1 per job; otherwise, default cpu is n_cpus / n_concurrent_trials- {"cpu": max(int(n_cpus / n_concurrent_trials), 1), "gpu": gpu_per_trial}- if gpu_per_trial == 0- else {"cpu": 1, "gpu": gpu_per_trial}+ (+ {+ "cpu": max(int((n_cpus - 2) / 2 / n_concurrent_trials), 1),+ "gpu": gpu_per_trial,+ }+ if gpu_per_trial == 0+ else {"cpu": 1, "gpu": gpu_per_trial}+ )if n_jobs < 0else {"cpu": n_jobs, "gpu": gpu_per_trial})+if isinstance(X_train, ray.ObjectRef):X_train = ray.get(X_train)elif isinstance(dataframe, ray.ObjectRef):@@ -2131,7 +2108,11 @@ class AutoML(BaseEstimator):))if "auto" == metric:- if "binary" in self._state.task:+ if _is_nlp_task(self._state.task):+ from .nlp.utils import load_default_huggingface_metric_for_task++ metric = load_default_huggingface_metric_for_task(self._state.task)+ elif "binary" in self._state.task:metric = "roc_auc"elif "multi" in self._state.task:metric = "log_loss"@@ -2139,17 +2120,9 @@ class AutoML(BaseEstimator):metric = "mape"elif self._state.task == "rank":metric = "ndcg"- elif _is_nlp_task(self._state.task):- from .nlp.utils import load_default_huggingface_metric_for_task-- metric = load_default_huggingface_metric_for_task(self._state.task)else:metric = "r2"- if _is_nlp_task(self._state.task):- self._state.fit_kwargs["metric"] = metric- self._state.fit_kwargs["use_ray"] = self._use_ray-self._state.metric = metricdef is_to_reverse_metric(metric, task):@@ -2355,6 +2328,14 @@ class AutoML(BaseEstimator):elif "random" == self._hpo_method:from ray.tune.suggest import BasicVariantGenerator as SearchAlgofrom ray.tune.sample import Domain+ elif "optuna" == self._hpo_method:+ try:+ from ray import __version__ as ray_version++ assert ray_version >= "1.0.0"+ from ray.tune.suggest.optuna import OptunaSearch as SearchAlgo+ except (ImportError, AssertionError):+ from .searcher.suggestion import OptunaSearch as SearchAlgoelse:raise NotImplementedError(f"hpo_method={self._hpo_method} is not recognized. "@@ -2382,24 +2363,48 @@ class AutoML(BaseEstimator):else:self._state.time_from_start = time.time() - self._start_time_flagtime_left = self._state.time_budget - self._state.time_from_start- search_alg = SearchAlgo(- metric="val_loss",- space=space,- low_cost_partial_config=self.low_cost_partial_config,- points_to_evaluate=self.points_to_evaluate,- cat_hp_cost=self.cat_hp_cost,- resource_attr=self.resource_attr,- min_resource=self.min_resource,- max_resource=self.max_resource,- config_constraints=[- (partial(size, self._state), "<=", self._mem_thres)- ],- metric_constraints=self.metric_constraints,- seed=self._seed,- time_budget_s=time_left,- )+ if self._hpo_method != "optuna":+ search_alg = SearchAlgo(+ metric="val_loss",+ space=space,+ low_cost_partial_config=self.low_cost_partial_config,+ points_to_evaluate=self.points_to_evaluate,+ cat_hp_cost=self.cat_hp_cost,+ resource_attr=self.resource_attr,+ min_resource=self.min_resource,+ max_resource=self.max_resource,+ config_constraints=[+ (partial(size, self._state), "<=", self._mem_thres)+ ],+ metric_constraints=self.metric_constraints,+ seed=self._seed,+ time_budget_s=time_left,+ )+ else:+ # if self._hpo_method is bo, sometimes the search space and the initial config dimension do not match+ # need to remove the extra keys from the search space to be consistent with the initial config+ converted_space = SearchAlgo.convert_search_space(space)++ removed_keys = set(space.keys()).difference(converted_space.keys())+ new_points_to_evaluate = []+ for idx in range(len(self.points_to_evaluate)):+ r = self.points_to_evaluate[idx].copy()+ for each_key in removed_keys:+ r.pop(each_key)+ new_points_to_evaluate.append(r)++ search_alg = SearchAlgo(+ metric="val_loss",+ mode="min",+ points_to_evaluate=[+ p+ for p in new_points_to_evaluate+ if len(p) == len(converted_space)+ ],+ )search_alg = ConcurrencyLimiter(search_alg, self._n_concurrent_trials)resources_per_trial = self._state.resources_per_trial+analysis = ray.tune.run(self.trainable,search_alg=search_alg,@@ -2413,6 +2418,7 @@ class AutoML(BaseEstimator):raise_on_failed_trial=False,keep_checkpoints_num=1,checkpoint_score_attr="min-val_loss",+ **self._use_ray if isinstance(self._use_ray, dict) else {},)# logger.info([trial.last_result for trial in analysis.trials])trials = sorted(@@ -2579,6 +2585,7 @@ class AutoML(BaseEstimator):if isinstance(search_state.init_config, list)else [search_state.init_config])+low_cost_partial_config = search_state.low_cost_partial_configif self._hpo_method in ("bs", "cfo", "grid", "cfocat", "random"):algo = SearchAlgo(@@ -2598,6 +2605,20 @@ class AutoML(BaseEstimator):seed=self._seed,)else:+ # if self._hpo_method is bo, sometimes the search space and the initial config dimension do not match+ # need to remove the extra keys from the search space to be consistent with the initial config+ converted_space = SearchAlgo.convert_search_space(search_space)+ removed_keys = set(search_space.keys()).difference(+ converted_space.keys()+ )+ new_points_to_evaluate = []+ for idx in range(len(points_to_evaluate)):+ r = points_to_evaluate[idx].copy()+ for each_key in removed_keys:+ r.pop(each_key)+ new_points_to_evaluate.append(r)+ points_to_evaluate = new_points_to_evaluate+algo = SearchAlgo(metric="val_loss",mode="min",diff --git a/flaml/ml.py b/flaml/ml.pyindex 146fe91ac..5e5c82d06 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -397,6 +397,7 @@ def get_val_loss(# fit_kwargs['groups_val'] = groups_val# fit_kwargs['X_val'] = X_val# fit_kwargs['y_val'] = y_val+estimator.fit(X_train, y_train, budget, **fit_kwargs)val_loss, metric_for_logging, pred_time, _ = _eval_estimator(config,@@ -561,6 +562,10 @@ def compute_estimator(task=task,n_jobs=n_jobs,)++ if isinstance(estimator, TransformersEstimator):+ fit_kwargs["metric"] = eval_metric+if "holdout" == eval_method:val_loss, metric_for_logging, train_time, pred_time = get_val_loss(config_dic,@@ -604,6 +609,7 @@ def train_estimator(estimator_class=None,budget=None,fit_kwargs={},+ eval_metric=None,):start_time = time.time()estimator_class = estimator_class or get_estimator_class(task, estimator_name)@@ -612,6 +618,9 @@ def train_estimator(task=task,n_jobs=n_jobs,)+ if isinstance(estimator, TransformersEstimator):+ fit_kwargs["metric"] = eval_metric+if X_train is not None:train_time = estimator.fit(X_train, y_train, budget, **fit_kwargs)else:diff --git a/flaml/model.py b/flaml/model.pyindex 9e6c20a1a..d0067ddf3 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -197,7 +197,7 @@ class BaseEstimator:train_time = self._fit(X_train, y_train, **kwargs)return train_time- def predict(self, X):+ def predict(self, X, **kwargs):"""Predict label from features.Args:@@ -216,7 +216,7 @@ class BaseEstimator:)return np.ones(X.shape[0])- def predict_proba(self, X):+ def predict_proba(self, X, **kwargs):"""Predict the probability of each class from features.Only works for classification problems@@ -325,7 +325,7 @@ class TransformersEstimator(BaseEstimator):},"num_train_epochs": {"domain": tune.loguniform(lower=0.1, upper=10.0),- "init_value": 3,+ "init_value": 1,},"per_device_train_batch_size": {"domain": tune.choice([4, 8, 16, 32]),@@ -344,33 +344,38 @@ class TransformersEstimator(BaseEstimator):"init_value": 1e-6,},"seed": {"domain": tune.choice(list(range(40, 45))), "init_value": 42},- "global_max_steps": {"domain": sys.maxsize, "init_value": sys.maxsize},+ "global_max_steps": {+ "domain": sys.maxsize,+ "init_value": sys.maxsize,+ },}- if task in NLG_TASKS:- search_space_dict["generation_num_beams"] = {- "domain": tune.randint(2, 5),- "init_value": 3,- }- search_space_dict["generation_max_length"] = {- "domain": tune.choice([16, 32, 64, 128]),- "init_value": 64,- }-return search_space_dict- def _init_hpo_args(self, automl_fit_kwargs: dict = None):- from .nlp.utils import HPOArgs+ def _init_hf_args(self, automl_fit_kwargs: dict = None):+ from .nlp.utils import HFArgs- custom_hpo_args = HPOArgs()- for key, val in automl_fit_kwargs["custom_hpo_args"].items():+ hf_args = HFArgs()+ for key, val in automl_fit_kwargs["hf_args"].items():assert (- key in custom_hpo_args.__dict__- ), "The specified key {} is not in the argument list of flaml.nlp.utils::HPOArgs".format(+ key in hf_args.__dict__+ ), "The specified key {} is not in the argument list of flaml.nlp.utils::HFArgs".format(key)- setattr(custom_hpo_args, key, val)- self.custom_hpo_args = custom_hpo_args+ setattr(hf_args, key, val)+ self.hf_args = hf_args++ def _update_hf_args(self, automl_pred_kwargs: dict = None):+ if automl_pred_kwargs:+ hf_args = automl_pred_kwargs.get("hf_args")+ if hf_args:+ for key, val in hf_args.items():+ assert (+ key in self.hf_args.__dict__+ ), "The specified key {} is not in the argument list of flaml.nlp.utils::HFArgs".format(+ key+ )+ setattr(self.hf_args, key, val)def _preprocess(self, X, y=None, **kwargs):from .nlp.utils import tokenize_text, is_a_list_of_str@@ -383,7 +388,7 @@ class TransformersEstimator(BaseEstimator):X=X,Y=y,task=self._task,- custom_hpo_args=self.custom_hpo_args,+ hf_args=self.hf_args,tokenizer=self._tokenizer,)else:@@ -392,12 +397,63 @@ class TransformersEstimator(BaseEstimator):def _model_init(self, num_labels, per_model_config):from .nlp.utils import load_model- return load_model(- checkpoint_path=self.custom_hpo_args.model_path,+ this_model = load_model(+ checkpoint_path=self.hf_args.model_path,task=self._task,num_labels=num_labels,per_model_config=per_model_config,)+ return this_model++ def _get_training_args(self, local_rank=-1):+ import transformers++ if self._task in NLG_TASKS:+ self._training_args_config["predict_with_generate"] = True++ if transformers.__version__.startswith("3"):+ training_args = self._TrainingArguments(+ report_to=[],+ output_dir=self._trial_dir,+ do_train=True,+ do_eval=True,+ eval_steps=self._ckpt_freq,+ evaluate_during_training=True,+ save_steps=self._ckpt_freq,+ logging_steps=self._ckpt_freq,+ save_total_limit=0,+ metric_for_best_model="loss",+ fp16=self.hf_args.fp16+ if self._kwargs.get("gpu_per_trial") > 0+ else False,+ no_cuda=True if self._kwargs.get("gpu_per_trial") == 0 else False,+ local_rank=local_rank,+ per_device_eval_batch_size=self.hf_args.per_device_eval_batch_size,+ **self._training_args_config,+ )+ else:+ from transformers import IntervalStrategy++ training_args = self._TrainingArguments(+ report_to=[],+ output_dir=self._trial_dir,+ do_train=True,+ do_eval=True,+ eval_steps=self._ckpt_freq,+ logging_steps=self._ckpt_freq,+ evaluation_strategy=IntervalStrategy.STEPS,+ save_steps=self._ckpt_freq,+ save_total_limit=0,+ metric_for_best_model="loss",+ fp16=self.hf_args.fp16+ if self._kwargs.get("gpu_per_trial") > 0+ else False,+ local_rank=local_rank,+ no_cuda=True if self._kwargs.get("gpu_per_trial") == 0 else False,+ per_device_eval_batch_size=self.hf_args.per_device_eval_batch_size,+ **self._training_args_config,+ )+ return training_argsdef fit(self, X_train: DataFrame, y_train: Series, budget=None, **kwargs):import transformers@@ -411,18 +467,11 @@ class TransformersEstimator(BaseEstimator):from .nlp.utils import (get_num_labels,separate_config,- load_model,compute_checkpoint_freq,- get_trial_fold_name,+ Counter,date_str,)- # TODO: if self._task == QUESTIONANSWERING, uncomment the code below (add indentation before- # from .nlp.huggingface.trainer import TrainerForAuto)-- # if self._task in NLG_TASKS:- # from .nlp.huggingface.trainer import Seq2SeqTrainerForAuto as TrainerForAuto- # else:from .nlp.huggingface.trainer import TrainerForAutofrom .nlp.huggingface.data_collator import DataCollatorForAutofrom .nlp.utils import get_auto_tokenizer@@ -462,13 +511,22 @@ class TransformersEstimator(BaseEstimator):set_seed(self.params.get("seed", self._TrainingArguments.seed))- self._init_hpo_args(kwargs)+ self._init_hf_args(kwargs)self._tokenizer = get_auto_tokenizer(- self.custom_hpo_args.model_path, self._task+ self.hf_args.tokenizer_model_path+ if self.hf_args.tokenizer_model_path+ else self.hf_args.model_path,+ self._task,)self._metric = kwargs["metric"]- self.use_ray = kwargs.get("use_ray")++ try:+ from ray.tune import is_session_enabled++ self.use_ray = is_session_enabled()+ except ImportError:+ self.use_ray = FalseX_val = kwargs.get("X_val")y_val = kwargs.get("y_val")@@ -498,70 +556,41 @@ class TransformersEstimator(BaseEstimator):eval_dataset = Nonenum_labels = get_num_labels(self._task, self._y_train)- training_args_config, per_model_config = separate_config(+ self._training_args_config, self._per_model_config = separate_config(self.params, self._task)- ckpt_freq = compute_checkpoint_freq(+ self._ckpt_freq = compute_checkpoint_freq(train_data_size=len(self._X_train),- custom_hpo_args=self.custom_hpo_args,- num_train_epochs=training_args_config.get(+ hf_args=self.hf_args,+ num_train_epochs=self._training_args_config.get("num_train_epochs", self._TrainingArguments.num_train_epochs),- batch_size=training_args_config.get(+ batch_size=self._training_args_config.get("per_device_train_batch_size",self._TrainingArguments.per_device_train_batch_size,),)- local_dir = os.path.join(- self.custom_hpo_args.output_dir, "train_{}".format(date_str())- )+ local_dir = os.path.join(self.hf_args.output_dir, "train_{}".format(date_str()))- if not self.use_ray:- # if self.params = {}, don't include configuration in trial fold name- trial_dir = get_trial_fold_name(local_dir, self.params, self.trial_id)- else:+ if self.use_ray is True:import ray- trial_dir = ray.tune.get_trial_dir()-- if transformers.__version__.startswith("3"):- training_args = self._TrainingArguments(- report_to=[],- output_dir=trial_dir,- do_train=True,- do_eval=True,- eval_steps=ckpt_freq,- evaluate_during_training=True,- save_steps=ckpt_freq,- logging_steps=ckpt_freq,- save_total_limit=0,- metric_for_best_model="loss",- fp16=self.custom_hpo_args.fp16,- **training_args_config,- )+ self._trial_dir = ray.tune.get_trial_dir()else:- from transformers import IntervalStrategy-- training_args = self._TrainingArguments(- report_to=[],- output_dir=trial_dir,- do_train=True,- do_eval=True,- per_device_eval_batch_size=1,- eval_steps=ckpt_freq,- logging_steps=ckpt_freq,- evaluation_strategy=IntervalStrategy.STEPS,- save_steps=ckpt_freq,- save_total_limit=0,- metric_for_best_model="loss",- fp16=self.custom_hpo_args.fp16,- **training_args_config,+ # if self.params = {}, don't include configuration in trial fold name+ self._trial_dir = Counter.get_trial_fold_name(+ local_dir, self.params, self.trial_id)+ self._kwargs = kwargs+ self._num_labels = num_labels++ training_args = self._get_training_args(local_rank=-1)+self._trainer = TrainerForAuto(args=training_args,- model_init=partial(self._model_init, num_labels, per_model_config),+ model_init=partial(self._model_init, num_labels, self._per_model_config),train_dataset=train_dataset,eval_dataset=eval_dataset,tokenizer=self._tokenizer,@@ -575,28 +604,33 @@ class TransformersEstimator(BaseEstimator):callbacks=[EarlyStoppingCallbackForAuto],)- setattr(self._trainer, "_use_ray", self.use_ray)if self._task in NLG_TASKS:setattr(self._trainer, "_is_seq2seq", True)- if kwargs.get("gpu_per_trial"):- self._trainer.args._n_gpu = kwargs.get("gpu_per_trial")++ gpu_per_trial = kwargs.get("gpu_per_trial", None)+ if gpu_per_trial:+ tmp_cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "")+ self._trainer.args._n_gpu = gpu_per_trial+ # if gpu_per_trial == 0:+ # os.environ["CUDA_VISIBLE_DEVICES"] = ""+ if tmp_cuda_visible_devices.count(",") != gpu_per_trial - 1:+ os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(+ [str(x) for x in range(gpu_per_trial)]+ )++ import time++ start_time = time.time()self._trainer.train()- self.params[self.ITER_HP] = self._trainer.state.global_step- self._checkpoint_path = self._select_checkpoint(self._trainer)+ if gpu_per_trial:+ os.environ["CUDA_VISIBLE_DEVICES"] = tmp_cuda_visible_devices- self._kwargs = kwargs- self._num_labels = num_labels- self._per_model_config = per_model_config- self._training_args_config = training_args_config+ self.params[self.ITER_HP] = self._trainer.state.global_step+ self._checkpoint_path = self._select_checkpoint(self._trainer)self._ckpt_remains = list(self._trainer.ckpt_to_metric.keys())- self._model = load_model(- checkpoint_path=self._checkpoint_path,- task=self._task,- num_labels=self._num_labels,- per_model_config=self._per_model_config,- )+if hasattr(self._trainer, "intermediate_results"):self.intermediate_results = [x[1]@@ -605,6 +639,7 @@ class TransformersEstimator(BaseEstimator):)]self._trainer = None+ return time.time() - start_timedef _delete_one_ckpt(self, ckpt_location):if self.use_ray is False:@@ -689,16 +724,21 @@ class TransformersEstimator(BaseEstimator):from datasets import Datasetfrom .nlp.huggingface.trainer import TrainerForAutofrom .nlp.huggingface.data_collator import DataCollatorForPredict+ from .nlp.utils import load_modelX_test, _ = self._preprocess(X_test, **self._kwargs)test_dataset = Dataset.from_pandas(X_test)- training_args = self._TrainingArguments(- per_device_eval_batch_size=1,- output_dir=self.custom_hpo_args.output_dir,- **self._training_args_config,++ this_model = load_model(+ checkpoint_path=self._checkpoint_path,+ task=self._task,+ num_labels=self._num_labels,+ per_model_config=self._per_model_config,)- self._trainer = TrainerForAuto(- model=self._model,+ training_args = self._get_training_args(local_rank=-1)++ new_trainer = TrainerForAuto(+ model=this_model,args=training_args,data_collator=DataCollatorForPredict(tokenizer=self._tokenizer,@@ -708,31 +748,36 @@ class TransformersEstimator(BaseEstimator):else None,compute_metrics=self._compute_metrics_by_dataset_name,)- return test_dataset, training_args+ if self._task in NLG_TASKS:+ setattr(new_trainer, "_is_seq2seq", True)+ return new_trainer, test_dataset, training_args- def predict_proba(self, X):+ def predict_proba(self, X, **kwargs):+ self._update_hf_args(kwargs)assert (self._task in CLASSIFICATION), "predict_proba() only for classification tasks."- test_dataset, _ = self._init_model_for_predict(X)- predictions = self._trainer.predict(test_dataset)- if self.use_ray is True:- self._trainer = None+ new_trainer, test_dataset, _ = self._init_model_for_predict(X)+ predictions = new_trainer.predict(test_dataset)return predictions.predictions- def predict(self, X):- test_dataset, training_args = self._init_model_for_predict(X)+ def predict(self, X, **kwargs):+ import transformers++ transformers.logging.set_verbosity_error()++ self._update_hf_args(kwargs)+ new_trainer, test_dataset, training_args = self._init_model_for_predict(X)+if self._task not in NLG_TASKS:- predictions = self._trainer.predict(test_dataset)+ predictions = new_trainer.predict(test_dataset)else:- predictions = self._trainer.predict(+ predictions = new_trainer.predict(test_dataset,- max_length=training_args.generation_max_length,- num_beams=training_args.generation_num_beams,+ metric_key_prefix="predict",)- if self.use_ray is True:- self._trainer = None+if self._task == SEQCLASSIFICATION:return np.argmax(predictions.predictions, axis=1)elif self._task == SEQREGRESSION:@@ -740,10 +785,8 @@ class TransformersEstimator(BaseEstimator):elif self._task == TOKENCLASSIFICATION:return np.argmax(predictions.predictions, axis=2)elif self._task == SUMMARIZATION:- if isinstance(predictions.predictions, tuple):- predictions = np.argmax(predictions.predictions[0], axis=2)decoded_preds = self._tokenizer.batch_decode(- predictions, skip_special_tokens=True+ predictions.predictions, skip_special_tokens=True)return decoded_predselif self._task == MULTICHOICECLASSIFICATION:@@ -1121,7 +1164,7 @@ class XGBoostEstimator(SKLearnEstimator):train_time = time.time() - start_timereturn train_time- def predict(self, X):+ def predict(self, X, **kwargs):import xgboost as xgbif not issparse(X):@@ -1617,7 +1660,7 @@ class Prophet(SKLearnEstimator):self._model = modelreturn train_time- def predict(self, X):+ def predict(self, X, **kwargs):if isinstance(X, int):raise ValueError("predict() with steps is only supported for arima/sarimax."@@ -1697,7 +1740,7 @@ class ARIMA(Prophet):self._model = modelreturn train_time- def predict(self, X):+ def predict(self, X, **kwargs):if self._model is not None:if isinstance(X, int):forecast = self._model.forecast(steps=X)@@ -1894,7 +1937,7 @@ class TS_SKLearn(SKLearnEstimator):train_time = time.time() - current_timereturn train_time- def predict(self, X):+ def predict(self, X, **kwargs):if self._model is not None:X = self.transform_X(X)X = self._preprocess(X)diff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.pyindex 22bd25faa..8604be28d 100644--- a/flaml/nlp/utils.py+++ b/flaml/nlp/utils.py@@ -2,6 +2,7 @@ import argparsefrom dataclasses import dataclass, fieldfrom itertools import chainfrom typing import Dict, Any+import numpy as npfrom ..data import (SUMMARIZATION,@@ -20,61 +21,54 @@ def load_default_huggingface_metric_for_task(task):elif task == SEQREGRESSION:return "r2"elif task == SUMMARIZATION:- return "rouge"+ return "rouge1"elif task == MULTICHOICECLASSIFICATION:return "accuracy"elif task == TOKENCLASSIFICATION:return "seqeval"-global tokenized_column_names---def get_auto_tokenizer(model_path, task):+def get_auto_tokenizer(tokenizer_model_path, task):from transformers import AutoTokenizerif task == SUMMARIZATION:return AutoTokenizer.from_pretrained(- model_path, # 'roberta-base'+ pretrained_model_name_or_path=tokenizer_model_path,cache_dir=None,use_fast=True,revision="main",use_auth_token=None,)else:- return AutoTokenizer.from_pretrained(model_path, use_fast=True)+ return AutoTokenizer.from_pretrained(tokenizer_model_path, use_fast=True)-def tokenize_text(X, Y=None, task=None, custom_hpo_args=None, tokenizer=None):+def tokenize_text(X, Y=None, task=None, hf_args=None, tokenizer=None):if task in (SEQCLASSIFICATION, SEQREGRESSION):X_tokenized = tokenize_onedataframe(X,tokenizer=tokenizer,task=task,- custom_hpo_args=custom_hpo_args,+ hf_args=hf_args,prefix_str="",)return X_tokenized, Noneelif task == TOKENCLASSIFICATION:return tokenize_text_tokclassification(- X, Y, tokenizer=tokenizer, custom_hpo_args=custom_hpo_args+ X, Y, tokenizer=tokenizer, hf_args=hf_args)elif task in NLG_TASKS:- return tokenize_seq2seq(- X, Y, tokenizer=tokenizer, task=task, custom_hpo_args=custom_hpo_args- )+ return tokenize_seq2seq(X, Y, tokenizer=tokenizer, task=task, hf_args=hf_args)elif task == MULTICHOICECLASSIFICATION:- return tokenize_text_multiplechoice(- X, tokenizer=tokenizer, custom_hpo_args=custom_hpo_args- )+ return tokenize_text_multiplechoice(X, tokenizer=tokenizer, hf_args=hf_args)-def tokenize_seq2seq(X, Y, tokenizer, task=None, custom_hpo_args=None):+def tokenize_seq2seq(X, Y, tokenizer, task=None, hf_args=None):model_inputs = tokenize_onedataframe(X,tokenizer=tokenizer,task=task,- custom_hpo_args=custom_hpo_args,+ hf_args=hf_args,prefix_str="summarize: ",)labels = None@@ -83,7 +77,7 @@ def tokenize_seq2seq(X, Y, tokenizer, task=None, custom_hpo_args=None):Y.to_frame(),tokenizer=tokenizer,task=task,- custom_hpo_args=custom_hpo_args,+ hf_args=hf_args,prefix_str="",)labels["label"] = [@@ -97,15 +91,18 @@ def tokenize_seq2seq(X, Y, tokenizer, task=None, custom_hpo_args=None):def tokenize_and_align_labels(- examples, tokenizer, custom_hpo_args=None, X_sent_key=None, Y_sent_key=None+ examples,+ tokenizer,+ hf_args=None,+ X_sent_key=None,+ Y_sent_key=None,+ return_column_name=False,):- global tokenized_column_names-tokenized_inputs = tokenizer([list(examples[X_sent_key])],padding="max_length",truncation=True,- max_length=custom_hpo_args.max_seq_length,+ max_length=hf_args.max_seq_length,# We use this argument because the texts in our dataset are lists of words (with a label for each word).is_split_into_words=True,)@@ -134,27 +131,37 @@ def tokenize_and_align_labels(# label_ids.append(b_to_i_label[label_to_id[label[word_idx]]])previous_word_idx = word_idxtokenized_inputs["label"] = label_ids- tokenized_column_names = sorted(tokenized_inputs.keys())- tokenized_input_and_labels = [tokenized_inputs[x] for x in tokenized_column_names]- for key_idx, each_key in enumerate(tokenized_column_names):+ tmp_column_names = sorted(tokenized_inputs.keys())+ tokenized_input_and_labels = [tokenized_inputs[x] for x in tmp_column_names]+ for key_idx, each_key in enumerate(tmp_column_names):if each_key != "label":tokenized_input_and_labels[key_idx] = tokenized_input_and_labels[key_idx][0]- return tokenized_input_and_labels+ if return_column_name:+ return tokenized_input_and_labels, tmp_column_names+ else:+ return tokenized_input_and_labels-def tokenize_text_tokclassification(X, Y, tokenizer, custom_hpo_args=None):+def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):import pandas as pd- global tokenized_column_namesif Y is not None:X_and_Y = pd.concat([X, Y.to_frame()], axis=1)X_key = list(X.keys())[0]Y_key = list(Y.to_frame().keys())[0]+ _, tokenized_column_names = tokenize_and_align_labels(+ X_and_Y.iloc[0],+ tokenizer=tokenizer,+ hf_args=hf_args,+ X_sent_key=X_key,+ Y_sent_key=Y_key,+ return_column_name=True,+ )X_and_Y_tokenized = X_and_Y.apply(lambda x: tokenize_and_align_labels(x,tokenizer=tokenizer,- custom_hpo_args=custom_hpo_args,+ hf_args=hf_args,X_sent_key=X_key,Y_sent_key=Y_key,),@@ -170,11 +177,21 @@ def tokenize_text_tokclassification(X, Y, tokenizer, custom_hpo_args=None):y_tokenized = X_and_Y_tokenized.iloc[:, label_idx]else:X_key = list(X.keys())[0]++ _, tokenized_column_names = tokenize_and_align_labels(+ X.iloc[0],+ tokenizer=tokenizer,+ hf_args=hf_args,+ X_sent_key=X_key,+ Y_sent_key=None,+ return_column_name=True,+ )+d = X.apply(lambda x: tokenize_and_align_labels(x,tokenizer=tokenizer,- custom_hpo_args=custom_hpo_args,+ hf_args=hf_args,X_sent_key=X_key,Y_sent_key=None,),@@ -192,28 +209,34 @@ def tokenize_onedataframe(X,tokenizer,task=None,- custom_hpo_args=None,+ hf_args=None,prefix_str=None,):import pandas- global tokenized_column_names-with tokenizer.as_target_tokenizer():+ _, tokenized_column_names = tokenize_row(+ dict(X.iloc[0]),+ tokenizer,+ prefix=(prefix_str,) if task is SUMMARIZATION else None,+ task=task,+ hf_args=hf_args,+ return_column_name=True,+ )d = X.apply(lambda x: tokenize_row(x,tokenizer,prefix=(prefix_str,) if task is SUMMARIZATION else None,task=task,- custom_hpo_args=custom_hpo_args,+ hf_args=hf_args,),axis=1,result_type="expand",)- X_tokenized = pandas.DataFrame(columns=tokenized_column_names)- X_tokenized[tokenized_column_names] = d- return X_tokenized+ X_tokenized = pandas.DataFrame(columns=tokenized_column_names)+ X_tokenized[tokenized_column_names] = d+ return X_tokenizeddef postprocess_text(preds, labels):@@ -230,35 +253,49 @@ def postprocess_text(preds, labels):return preds, labels-def tokenize_row(this_row, tokenizer, prefix=None, task=None, custom_hpo_args=None):- global tokenized_column_names+def tokenize_row(+ this_row,+ tokenizer,+ prefix=None,+ task=None,+ hf_args=None,+ return_column_name=False,+):assert (- "max_seq_length" in custom_hpo_args.__dict__+ "max_seq_length" in hf_args.__dict__), "max_seq_length must be provided for glue"if prefix:this_row = tuple(["".join(x) for x in zip(prefix, this_row)])+ # tokenizer.pad_token = tokenizer.eos_tokentokenized_example = tokenizer(*tuple(this_row),padding="max_length",- max_length=custom_hpo_args.max_seq_length,+ max_length=hf_args.max_seq_length,truncation=True,)if task in NLG_TASKS:tokenized_example["decoder_input_ids"] = tokenized_example["input_ids"]- tokenized_column_names = sorted(tokenized_example.keys())- return [tokenized_example[x] for x in tokenized_column_names]+ tmp_column_names = sorted(tokenized_example.keys())+ if return_column_name:+ return [tokenized_example[x] for x in tmp_column_names], tmp_column_names+ else:+ return [tokenized_example[x] for x in tmp_column_names]-def tokenize_text_multiplechoice(X, tokenizer, custom_hpo_args=None):+def tokenize_text_multiplechoice(X, tokenizer, hf_args=None):import pandas- global tokenized_column_names-t = X[["sent1", "sent2", "ending0", "ending1", "ending2", "ending3"]]+ _, tokenized_column_names = tokenize_swag(+ t.iloc[0],+ tokenizer=tokenizer,+ hf_args=hf_args,+ return_column_name=True,+ )d = t.apply(- lambda x: tokenize_swag(x, tokenizer, custom_hpo_args),+ lambda x: tokenize_swag(x, tokenizer=tokenizer, hf_args=hf_args),axis=1,result_type="expand",)@@ -269,9 +306,7 @@ def tokenize_text_multiplechoice(X, tokenizer, custom_hpo_args=None):return output, None-def tokenize_swag(this_row, tokenizer, custom_hpo_args=None):- global tokenized_column_names-+def tokenize_swag(this_row, tokenizer, hf_args=None, return_column_name=False):first_sentences = [[this_row["sent1"]] * 4]# get each 1st sentence, multiply to 4 sentencesquestion_headers = this_row["sent2"]@@ -289,11 +324,15 @@ def tokenize_swag(this_row, tokenizer, custom_hpo_args=None):tokenized_example = tokenizer(*tuple([first_sentences, second_sentences]),truncation=True,- max_length=custom_hpo_args.max_seq_length,+ max_length=hf_args.max_seq_length,padding=False,)- tokenized_column_names = sorted(tokenized_example.keys())- return [tokenized_example[x] for x in tokenized_column_names]+ tmp_column_names = sorted(tokenized_example.keys())++ if return_column_name:+ return [tokenized_example[x] for x in tmp_column_names], tmp_column_names+ else:+ return [tokenized_example[x] for x in tmp_column_names]def separate_config(config, task):@@ -333,7 +372,9 @@ def get_num_labels(task, y_train):def is_a_list_of_str(this_obj):- return isinstance(this_obj, list) and all(isinstance(x, str) for x in this_obj)+ return (isinstance(this_obj, list) or isinstance(this_obj, np.ndarray)) and all(+ isinstance(x, str) for x in this_obj+ )def _clean_value(value: Any) -> str:@@ -386,14 +427,19 @@ def get_logdir_name(dirname, local_dir):return logdir-def get_trial_fold_name(local_dir, trial_config, trial_id):- global counter- counter = counter + 1- experiment_tag = "{0}_{1}".format(str(counter), format_vars(trial_config))- logdir = get_logdir_name(- _generate_dirname(experiment_tag, trial_id=trial_id), local_dir- )- return logdir+class Counter:+ counter = 0++ @staticmethod+ def get_trial_fold_name(local_dir, trial_config, trial_id):+ Counter.counter += 1+ experiment_tag = "{0}_{1}".format(+ str(Counter.counter), format_vars(trial_config)+ )+ logdir = get_logdir_name(+ _generate_dirname(experiment_tag, trial_id=trial_id), local_dir+ )+ return logdirdef load_model(checkpoint_path, task, num_labels, per_model_config=None):@@ -499,7 +545,7 @@ def load_model(checkpoint_path, task, num_labels, per_model_config=None):def compute_checkpoint_freq(train_data_size,- custom_hpo_args,+ hf_args,num_train_epochs,batch_size,):@@ -508,7 +554,7 @@ def compute_checkpoint_freq(min(num_train_epochs, 1)* train_data_size/ batch_size- / custom_hpo_args.ckpt_per_epoch+ / hf_args.ckpt_per_epoch)+ 1)@@ -516,7 +562,7 @@ def compute_checkpoint_freq(@dataclass-class HPOArgs:+class HFArgs:"""The HPO setting.Args:output_dir (str): data root directory for outputing the log, etc.@@ -534,7 +580,12 @@ class HPOArgs:model_path: str = field(default="facebook/muppet-roberta-base",- metadata={"help": "model path model for HPO"},+ metadata={"help": "model path for HPO"},+ )++ tokenizer_model_path: str = field(+ default=None,+ metadata={"help": "tokenizer model path for HPO"},)fp16: bool = field(default=True, metadata={"help": "whether to use the FP16 mode"})@@ -552,12 +603,17 @@ class HPOArgs:ckpt_per_epoch: int = field(default=1, metadata={"help": "checkpoint per epoch"})+ per_device_eval_batch_size: int = field(+ default=1,+ metadata={"help": "per gpu evaluation batch size"},+ )+@staticmethoddef load_args():from dataclasses import fieldsarg_parser = argparse.ArgumentParser()- for each_field in fields(HPOArgs):+ for each_field in fields(HFArgs):print(each_field)arg_parser.add_argument("--" + each_field.name,diff --git a/flaml/training_log.py b/flaml/training_log.pyindex a71488f03..8f3fc246b 100644--- a/flaml/training_log.py+++ b/flaml/training_log.py@@ -79,7 +79,7 @@ class TrainingLogWriter(object):sample_size,):if self.file is None:- raise IOError("Call open() to open the outpute file first.")+ raise IOError("Call open() to open the output file first.")if validation_loss is None:raise ValueError("TEST LOSS NONE ERROR!!!")record = TrainingLogRecord(@@ -109,7 +109,7 @@ class TrainingLogWriter(object):def checkpoint(self):if self.file is None:- raise IOError("Call open() to open the outpute file first.")+ raise IOError("Call open() to open the output file first.")if self.current_best_loss_record_id is None:logger.warning("flaml.training_log: checkpoint() called before any record is written, skipped."diff --git a/notebook/automl_nlp.ipynb b/notebook/automl_nlp.ipynbnew file mode 100644index 000000000..c74d89c9d--- /dev/null+++ b/notebook/automl_nlp.ipynb@@ -0,0 +1,1522 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {+ "slideshow": {+ "slide_type": "slide"+ }+ },+ "source": [+ "Copyright (c) Microsoft Corporation. All rights reserved. \n",+ "\n",+ "Licensed under the MIT License.\n",+ "\n",+ "# FineTuning NLP Models with FLAML Library\n",+ "\n",+ "\n",+ "## 1. Introduction\n",+ "\n",+ "FLAML is a Python library (https://github.com/microsoft/FLAML) designed to automatically produce accurate machine learning models \n",+ "with low computational cost. It is fast and economical. The simple and lightweight design makes it easy to use and extend, such as adding new learners. FLAML can \n",+ "- serve as an economical AutoML engine,\n",+ "- be used as a fast hyperparameter tuning tool, or \n",+ "- be embedded in self-tuning software that requires low latency & resource in repetitive\n",+ " tuning tasks.\n",+ "\n",+ "In this notebook, we demonstrate how to use the FLAML library to fine tune an NLP language model with hyperparameter search. We have tested this notebook on a server with 4 NVidia V100 GPU (32GB) and 400GB CPU Ram.\n",+ "\n",+ "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `nlp,ray,notebook` and `blendsearch` option:\n",+ "```bash\n",+ "pip install flaml[nlp,ray,notebook,blendsearch];\n",+ "```"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Requirement already satisfied: flaml[blendsearch,nlp,notebook,ray] in /data/xliu127/projects/hyperopt/FLAML (0.10.0)\n",+ "Requirement already satisfied: NumPy>=1.16.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.22.3)\n",+ "Requirement already satisfied: lightgbm>=2.3.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.3.2)\n",+ "Requirement already satisfied: xgboost<=1.3.3,>=0.90 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.3.3)\n",+ "Requirement already satisfied: scipy>=1.4.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.8.0)\n",+ "Requirement already satisfied: pandas>=1.1.4 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.4.1)\n",+ "Requirement already satisfied: scikit-learn>=0.24 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.0.2)\n",+ "Requirement already satisfied: openml==0.10.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (0.10.2)\n",+ "Requirement already satisfied: jupyter in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.0.0)\n",+ "Requirement already satisfied: matplotlib in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.5.1)\n",+ "Requirement already satisfied: rgf-python in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.12.0)\n",+ "Requirement already satisfied: catboost>=0.26 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.0.4)\n",+ "Requirement already satisfied: optuna==2.8.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (2.8.0)\n",+ "Requirement already satisfied: transformers>=4.14 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (4.17.0)\n",+ "Requirement already satisfied: datasets in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.18.4)\n",+ "Requirement already satisfied: torch in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.11.0)\n",+ "Requirement already satisfied: seqeval in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.2.2)\n",+ "Requirement already satisfied: nltk in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (3.7)\n",+ "Requirement already satisfied: rouge_score in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (0.0.4)\n",+ "Requirement already satisfied: ray[tune]~=1.10 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from flaml[blendsearch,nlp,notebook,ray]) (1.11.0)\n",+ "Requirement already satisfied: liac-arff>=2.4.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.5.0)\n",+ "Requirement already satisfied: xmltodict in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (0.12.0)\n",+ "Requirement already satisfied: requests in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.27.1)\n",+ "Requirement already satisfied: python-dateutil in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.8.2)\n",+ "Requirement already satisfied: sqlalchemy>=1.1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.4.32)\n",+ "Requirement already satisfied: packaging>=20.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (21.3)\n",+ "Requirement already satisfied: colorlog in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (6.6.0)\n",+ "Requirement already satisfied: cmaes>=0.8.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (0.8.2)\n",+ "Requirement already satisfied: alembic in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.7.6)\n",+ "Requirement already satisfied: tqdm in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (4.63.0)\n",+ "Requirement already satisfied: cliff in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.10.1)\n",+ "Requirement already satisfied: six in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (1.16.0)\n",+ "Requirement already satisfied: graphviz in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (0.19.1)\n",+ "Requirement already satisfied: plotly in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (5.6.0)\n",+ "Requirement already satisfied: wheel in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from lightgbm>=2.3.1->flaml[blendsearch,nlp,notebook,ray]) (0.37.1)\n",+ "Requirement already satisfied: pytz>=2020.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from pandas>=1.1.4->flaml[blendsearch,nlp,notebook,ray]) (2021.3)\n",+ "Requirement already satisfied: attrs in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (21.4.0)\n",+ "Requirement already satisfied: protobuf>=3.15.3 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (3.19.4)\n",+ "Requirement already satisfied: pyyaml in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (6.0)\n",+ "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.0.3)\n",+ "Requirement already satisfied: grpcio<=1.43.0,>=1.28.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.43.0)\n",+ "Requirement already satisfied: click>=7.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (8.0.4)\n",+ "Requirement already satisfied: jsonschema in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (4.4.0)\n",+ "Requirement already satisfied: filelock in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (3.6.0)\n",+ "Requirement already satisfied: redis>=3.5.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (4.1.4)\n",+ "Requirement already satisfied: tensorboardX>=1.9 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (2.5)\n",+ "Requirement already satisfied: tabulate in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (0.8.9)\n",+ "Requirement already satisfied: joblib>=0.11 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from scikit-learn>=0.24->flaml[blendsearch,nlp,notebook,ray]) (1.1.0)\n",+ "Requirement already satisfied: threadpoolctl>=2.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from scikit-learn>=0.24->flaml[blendsearch,nlp,notebook,ray]) (3.1.0)\n",+ "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (0.4.0)\n",+ "Requirement already satisfied: tokenizers!=0.11.3,>=0.11.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (0.11.6)\n",+ "Requirement already satisfied: regex!=2019.12.17 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (2022.3.2)\n",+ "Requirement already satisfied: sacremoses in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from transformers>=4.14->flaml[blendsearch,nlp,notebook,ray]) (0.0.49)\n",+ "Requirement already satisfied: multiprocess in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (0.70.12.2)\n",+ "Requirement already satisfied: xxhash in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (3.0.0)\n",+ "Requirement already satisfied: dill in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (0.3.4)\n",+ "Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (7.0.0)\n",+ "Requirement already satisfied: fsspec[http]>=2021.05.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (2022.2.0)\n",+ "Requirement already satisfied: aiohttp in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (3.8.1)\n",+ "Requirement already satisfied: responses<0.19 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from datasets->flaml[blendsearch,nlp,notebook,ray]) (0.18.0)\n",+ "Requirement already satisfied: nbconvert in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.4.4)\n",+ "Requirement already satisfied: ipywidgets in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (7.6.5)\n",+ "Requirement already satisfied: ipykernel in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.9.2)\n",+ "Requirement already satisfied: notebook in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.4.10)\n",+ "Requirement already satisfied: qtconsole in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.2.2)\n",+ "Requirement already satisfied: jupyter-console in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.4.3)\n",+ "Requirement already satisfied: fonttools>=4.22.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (4.30.0)\n",+ "Requirement already satisfied: cycler>=0.10 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (0.11.0)\n",+ "Requirement already satisfied: kiwisolver>=1.0.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (1.4.0)\n",+ "Requirement already satisfied: pyparsing>=2.2.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (3.0.7)\n",+ "Requirement already satisfied: pillow>=6.2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from matplotlib->flaml[blendsearch,nlp,notebook,ray]) (9.0.1)\n",+ "Requirement already satisfied: absl-py in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from rouge_score->flaml[blendsearch,nlp,notebook,ray]) (1.0.0)\n",+ "Requirement already satisfied: typing-extensions in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from torch->flaml[blendsearch,nlp,notebook,ray]) (4.1.1)\n",+ "Requirement already satisfied: deprecated>=1.2.3 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from redis>=3.5.0->ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.2.13)\n",+ "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (1.26.8)\n",+ "Requirement already satisfied: certifi>=2017.4.17 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2021.10.8)\n",+ "Requirement already satisfied: charset-normalizer~=2.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (2.0.12)\n",+ "Requirement already satisfied: idna<4,>=2.5 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from requests->openml==0.10.2->flaml[blendsearch,nlp,notebook,ray]) (3.3)\n",+ "Requirement already satisfied: greenlet!=0.4.17 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from sqlalchemy>=1.1.0->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.1.2)\n",+ "Requirement already satisfied: aiosignal>=1.1.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (1.2.0)\n",+ "Requirement already satisfied: multidict<7.0,>=4.5 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (6.0.2)\n",+ "Requirement already satisfied: frozenlist>=1.1.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (1.3.0)\n",+ "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (4.0.2)\n",+ "Requirement already satisfied: yarl<2.0,>=1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from aiohttp->datasets->flaml[blendsearch,nlp,notebook,ray]) (1.7.2)\n",+ "Requirement already satisfied: Mako in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.2.0)\n",+ "Requirement already satisfied: importlib-resources in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (5.4.0)\n",+ "Requirement already satisfied: importlib-metadata in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (4.11.3)\n",+ "Requirement already satisfied: pbr!=2.1.0,>=2.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (5.8.1)\n",+ "Requirement already satisfied: cmd2>=1.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (2.4.0)\n",+ "Requirement already satisfied: PrettyTable>=0.7.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.2.0)\n",+ "Requirement already satisfied: stevedore>=2.0.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.5.0)\n",+ "Requirement already satisfied: autopage>=0.4.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (0.5.0)\n",+ "Requirement already satisfied: traitlets<6.0,>=5.1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.1.1)\n",+ "Requirement already satisfied: tornado<7.0,>=4.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (6.1)\n",+ "Requirement already satisfied: ipython>=7.23.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (8.1.1)\n",+ "Requirement already satisfied: matplotlib-inline<0.2.0,>=0.1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.1.3)\n",+ "Requirement already satisfied: nest-asyncio in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.5.4)\n",+ "Requirement already satisfied: psutil in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.9.0)\n",+ "Requirement already satisfied: debugpy<2.0,>=1.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.5.1)\n",+ "Requirement already satisfied: jupyter-client<8.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (7.1.2)\n",+ "Requirement already satisfied: widgetsnbextension~=3.5.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipywidgets->jupyter->flaml[blendsearch,nlp,notebook,ray]) (3.5.2)\n",+ "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipywidgets->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.0.2)\n",+ "Requirement already satisfied: ipython-genutils~=0.2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipywidgets->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.0)\n",+ "Requirement already satisfied: nbformat>=4.2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipywidgets->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.2.0)\n",+ "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jsonschema->ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (0.18.1)\n",+ "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter-console->jupyter->flaml[blendsearch,nlp,notebook,ray]) (3.0.27)\n",+ "Requirement already satisfied: pygments in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jupyter-console->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.11.2)\n",+ "Requirement already satisfied: jupyterlab-pygments in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.1.2)\n",+ "Requirement already satisfied: defusedxml in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.7.1)\n",+ "Requirement already satisfied: testpath in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.6.0)\n",+ "Requirement already satisfied: jinja2>=2.4 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (3.0.3)\n",+ "Requirement already satisfied: pandocfilters>=1.4.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.5.0)\n",+ "Requirement already satisfied: jupyter-core in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.9.2)\n",+ "Requirement already satisfied: entrypoints>=0.2.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.4)\n",+ "Requirement already satisfied: bleach in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.1.0)\n",+ "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.5.13)\n",+ "Requirement already satisfied: mistune<2,>=0.8.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.8.4)\n",+ "Requirement already satisfied: beautifulsoup4 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.10.0)\n",+ "Requirement already satisfied: argon2-cffi in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (21.3.0)\n",+ "Requirement already satisfied: prometheus-client in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.13.1)\n",+ "Requirement already satisfied: pyzmq>=17 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (22.3.0)\n",+ "Requirement already satisfied: Send2Trash>=1.8.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.8.0)\n",+ "Requirement already satisfied: terminado>=0.8.3 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.13.3)\n",+ "Requirement already satisfied: tenacity>=6.2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from plotly->catboost>=0.26->flaml[blendsearch,nlp,notebook,ray]) (8.0.1)\n",+ "Requirement already satisfied: qtpy in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from qtconsole->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.0.1)\n",+ "Requirement already satisfied: wcwidth>=0.1.7 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cmd2>=1.0.0->cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (0.2.5)\n",+ "Requirement already satisfied: pyperclip>=1.6 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cmd2>=1.0.0->cliff->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (1.8.2)\n",+ "Requirement already satisfied: wrapt<2,>=1.10 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from deprecated>=1.2.3->redis>=3.5.0->ray[tune]~=1.10->flaml[blendsearch,nlp,notebook,ray]) (1.14.0)\n",+ "Requirement already satisfied: zipp>=3.1.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from importlib-resources->alembic->optuna==2.8.0->flaml[blendsearch,nlp,notebook,ray]) (3.7.0)\n",+ "Requirement already satisfied: backcall in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.0)\n",+ "Requirement already satisfied: pexpect>4.3 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (4.8.0)\n",+ "Requirement already satisfied: decorator in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (5.1.1)\n",+ "Requirement already satisfied: jedi>=0.16 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.18.1)\n",+ "Requirement already satisfied: setuptools>=18.5 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (60.9.3)\n",+ "Requirement already satisfied: stack-data in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.0)\n",+ "Requirement already satisfied: pickleshare in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.7.5)\n",+ "Requirement already satisfied: MarkupSafe>=2.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jinja2>=2.4->nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.1.1)\n",+ "Requirement already satisfied: ptyprocess in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from terminado>=0.8.3->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.7.0)\n",+ "Requirement already satisfied: argon2-cffi-bindings in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from argon2-cffi->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (21.2.0)\n",+ "Requirement already satisfied: soupsieve>1.2 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from beautifulsoup4->nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.3.1)\n",+ "Requirement already satisfied: webencodings in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from bleach->nbconvert->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.5.1)\n",+ "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.8.3)\n",+ "Requirement already satisfied: cffi>=1.0.1 in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from argon2-cffi-bindings->argon2-cffi->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (1.15.0)\n",+ "Requirement already satisfied: pure-eval in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.2.2)\n",+ "Requirement already satisfied: executing in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (0.8.3)\n",+ "Requirement already satisfied: asttokens in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.0.5)\n",+ "Requirement already satisfied: pycparser in /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook->jupyter->flaml[blendsearch,nlp,notebook,ray]) (2.21)\n"+ ]+ }+ ],+ "source": [+ "!pip install flaml[nlp,ray,notebook,blendsearch];"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's run some examples. \n",+ "\n",+ "Note: throughout this notebook, you may see a few ModuleNotFoundErrors. As long as the cell successfully executes, you can ignore that error."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 2. Sentiment Classification Example\n",+ "### Load data and preprocess\n",+ "\n",+ "The Stanford Sentiment treebank (SST-2) dataset is a dataset for sentiment classification. First, let's load this dataset into pandas dataframes:"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",+ "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",+ "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"+ ]+ }+ ],+ "source": [+ "from datasets import load_dataset\n",+ "\n",+ "train_dataset = load_dataset(\"glue\", \"sst2\", split=\"train\").to_pandas().iloc[:10000]\n",+ "dev_dataset = load_dataset(\"glue\", \"sst2\", split=\"validation\").to_pandas().iloc[:10000]\n",+ "test_dataset = load_dataset(\"glue\", \"sst2\", split=\"test\").to_pandas()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Take a look at the first 5 examples of this dataset:"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/html": [+ "<div>\n",+ "<style scoped>\n",+ " .dataframe tbody tr th:only-of-type {\n",+ " vertical-align: middle;\n",+ " }\n",+ "\n",+ " .dataframe tbody tr th {\n",+ " vertical-align: top;\n",+ " }\n",+ "\n",+ " .dataframe thead th {\n",+ " text-align: right;\n",+ " }\n",+ "</style>\n",+ "<table border=\"1\" class=\"dataframe\">\n",+ " <thead>\n",+ " <tr style=\"text-align: right;\">\n",+ " <th></th>\n",+ " <th>sentence</th>\n",+ " <th>label</th>\n",+ " <th>idx</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th>0</th>\n",+ " <td>hide new secretions from the parental units</td>\n",+ " <td>0</td>\n",+ " <td>0</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th>1</th>\n",+ " <td>contains no wit , only labored gags</td>\n",+ " <td>0</td>\n",+ " <td>1</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th>2</th>\n",+ " <td>that loves its characters and communicates som...</td>\n",+ " <td>1</td>\n",+ " <td>2</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th>3</th>\n",+ " <td>remains utterly satisfied to remain the same t...</td>\n",+ " <td>0</td>\n",+ " <td>3</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th>4</th>\n",+ " <td>on the worst revenge-of-the-nerds clichés the ...</td>\n",+ " <td>0</td>\n",+ " <td>4</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n",+ "</div>"+ ],+ "text/plain": [+ " sentence label idx\n",+ "0 hide new secretions from the parental units 0 0\n",+ "1 contains no wit , only labored gags 0 1\n",+ "2 that loves its characters and communicates som... 1 2\n",+ "3 remains utterly satisfied to remain the same t... 0 3\n",+ "4 on the worst revenge-of-the-nerds clichés the ... 0 4"+ ]+ },+ "execution_count": 3,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "train_dataset.head(5)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Separate the data into X and y:"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [],+ "source": [+ "custom_sent_keys = [\"sentence\"] # specify the column names of the input sentences\n",+ "label_key = \"label\" # specify the column name of the label\n",+ "\n",+ "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n",+ "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n",+ "X_test = test_dataset[custom_sent_keys]"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Run FLAML"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/xgboost/compat.py:31: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",+ " from pandas import MultiIndex, Int64Index\n",+ "2022-03-19 13:59:38,805\tERROR services.py:1421 -- Failed to start the dashboard: Failed to start the dashboard, return code 1\n",+ "Failed to read dashboard log: [Errno 2] No such file or directory: '/tmp/ray/session_2022-03-19_13-59-37_048935_85869/logs/dashboard.log'\n",+ "\u001B[2m\u001B[33m(raylet)\u001B[0m Traceback (most recent call last):\n",+ "\u001B[2m\u001B[33m(raylet)\u001B[0m File \"/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/ray/dashboard/agent.py\", line 21, in <module>\n",+ "\u001B[2m\u001B[33m(raylet)\u001B[0m import ray.dashboard.utils as dashboard_utils\n",+ "\u001B[2m\u001B[33m(raylet)\u001B[0m File \"/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/ray/dashboard/utils.py\", line 15, in <module>\n",+ "\u001B[2m\u001B[33m(raylet)\u001B[0m import aioredis # noqa: F401\n",+ "\u001B[2m\u001B[33m(raylet)\u001B[0m ModuleNotFoundError: No module named 'aioredis'\n"+ ]+ }+ ],+ "source": [+ "''' import AutoML class from flaml package '''\n",+ "from flaml import AutoML\n",+ "automl = AutoML()\n",+ "\n",+ "import ray\n",+ "if not ray.is_initialized():\n",+ " ray.init() "+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "\n",+ "automl_settings = {\n",+ " \"time_budget\": 500, # setting the time budget\n",+ " \"task\": \"seq-classification\", # setting the task as seq-classification\n",+ " \"hf_args\": {\n",+ " \"output_dir\": \"data/output/\", # setting the output directory\n",+ " \"ckpt_per_epoch\": 1, # setting the number of checkoints per epoch\n",+ " \"model_path\": \"google/electra-base-discriminator\",\n",+ " \"per_device_eval_batch_size\": 16 # the batch size for validation (inference)\n",+ " },\n",+ " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n",+ " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n",+ " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n",+ " \"use_ray\": True, # set whether to use Ray\n",+ " \"n_concurrent_trials\": 4,\n",+ " \"keep_search_state\": True, # keeping the search state\n",+ "}"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/html": [+ "== Status ==<br>Current time: 2022-03-19 14:08:10 (running for 00:08:23.62)<br>Memory usage on this node: 26.2/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/250.17 GiB heap, 0.0/111.21 GiB objects (0.0/1.0 accelerator_type:V100)<br>Current best trial: 8275ca0a with val_loss=0.059633027522935755 and parameters={'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}<br>Result logdir: /home/xliu127/ray_results/train_2022-03-19_13-59-47<br>Number of trials: 14/1000000 (14 TERMINATED)<br><br>"+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m {'loss': 0.688, 'learning_rate': 5.91472422187606e-05, 'epoch': 6.0}\n",+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m {'eval_loss': 0.6975926160812378, 'eval_automl_metric': 0.49082568807339455, 'eval_runtime': 0.9885, 'eval_samples_per_second': 882.13, 'eval_steps_per_second': 55.639, 'epoch': 6.0}\n",+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m {'eval_loss': 0.6959978938102722, 'eval_automl_metric': 0.49082568807339455, 'eval_runtime': 0.9612, 'eval_samples_per_second': 907.223, 'eval_steps_per_second': 57.222, 'epoch': 6.04}\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'loss': 0.133, 'learning_rate': 1.9966093201540806e-05, 'epoch': 3.0}\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'eval_loss': 0.29954516887664795, 'eval_automl_metric': 0.06880733944954132, 'eval_runtime': 0.9772, 'eval_samples_per_second': 892.372, 'eval_steps_per_second': 56.285, 'epoch': 3.0}\n",+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m {'eval_loss': 0.6959978938102722, 'eval_automl_metric': 0.49082568807339455, 'eval_runtime': 0.9707, 'eval_samples_per_second': 898.292, 'eval_steps_per_second': 56.658, 'epoch': 6.04}\n",+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m {'train_runtime': 510.9986, 'train_samples_per_second': 142.692, 'train_steps_per_second': 17.838, 'train_loss': 0.6829553683756834, 'epoch': 6.04}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m Num examples = 872\n",+ "\u001B[2m\u001B[36m(train pid=86265)\u001B[0m Batch size = 16\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'loss': 0.2158, 'learning_rate': 5.454224012822699e-06, 'epoch': 3.0}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.21694229543209076, 'eval_automl_metric': 0.06422018348623848, 'eval_runtime': 0.9902, 'eval_samples_per_second': 880.648, 'eval_steps_per_second': 55.545, 'epoch': 3.0}\n",+ "\u001B[2m\u001B[36m(train pid=86269)\u001B[0m {'eval_loss': 0.5855236649513245, 'eval_automl_metric': 0.18463302752293576, 'eval_runtime': 0.9877, 'eval_samples_per_second': 882.845, 'eval_steps_per_second': 55.684, 'epoch': 4.53}\n",+ "\u001B[2m\u001B[36m(train pid=86269)\u001B[0m {'train_runtime': 373.6856, 'train_samples_per_second': 121.13, 'train_steps_per_second': 15.144, 'train_loss': 0.4671156674776214, 'epoch': 4.53}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86269)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86269)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86269)\u001B[0m Num examples = 872\n",+ "\u001B[2m\u001B[36m(train pid=86269)\u001B[0m Batch size = 16\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'loss': 0.0655, 'learning_rate': 1.1451941740938257e-05, 'epoch': 4.01}\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'eval_loss': 0.32505378127098083, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 0.9623, 'eval_samples_per_second': 906.171, 'eval_steps_per_second': 57.155, 'epoch': 4.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'loss': 0.1585, 'learning_rate': 4.673980815375941e-06, 'epoch': 4.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.26288139820098877, 'eval_automl_metric': 0.06422018348623848, 'eval_runtime': 0.9914, 'eval_samples_per_second': 879.599, 'eval_steps_per_second': 55.479, 'epoch': 4.01}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:09:46,075\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'eval_loss': 0.37855082750320435, 'eval_automl_metric': 0.06536697247706424, 'eval_runtime': 0.9622, 'eval_samples_per_second': 906.215, 'eval_steps_per_second': 57.158, 'epoch': 5.0}\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'loss': 0.0323, 'learning_rate': 2.937790280335705e-06, 'epoch': 5.01}\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'eval_loss': 0.38091975450515747, 'eval_automl_metric': 0.06536697247706424, 'eval_runtime': 0.9618, 'eval_samples_per_second': 906.667, 'eval_steps_per_second': 57.187, 'epoch': 5.01}\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'eval_loss': 0.4038548171520233, 'eval_automl_metric': 0.06995412844036697, 'eval_runtime': 0.9613, 'eval_samples_per_second': 907.15, 'eval_steps_per_second': 57.217, 'epoch': 5.35}\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m {'train_runtime': 240.3679, 'train_samples_per_second': 222.334, 'train_steps_per_second': 13.9, 'train_loss': 0.1605183750667532, 'epoch': 5.35}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m Num examples = 872\n",+ "\u001B[2m\u001B[36m(train pid=86255)\u001B[0m Batch size = 16\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'loss': 0.1288, 'learning_rate': 3.893737617929184e-06, 'epoch': 5.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.2783028781414032, 'eval_automl_metric': 0.06536697247706424, 'eval_runtime': 0.9904, 'eval_samples_per_second': 880.481, 'eval_steps_per_second': 55.535, 'epoch': 5.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'loss': 0.0921, 'learning_rate': 3.114740815366527e-06, 'epoch': 6.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.2939322590827942, 'eval_automl_metric': 0.06422018348623848, 'eval_runtime': 0.9917, 'eval_samples_per_second': 879.284, 'eval_steps_per_second': 55.459, 'epoch': 6.01}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:11:16,956\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'loss': 0.0749, 'learning_rate': 2.3357440128038706e-06, 'epoch': 7.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.3375593423843384, 'eval_automl_metric': 0.0665137614678899, 'eval_runtime': 1.0158, 'eval_samples_per_second': 858.42, 'eval_steps_per_second': 54.143, 'epoch': 7.01}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:12:16,966\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'loss': 0.06, 'learning_rate': 1.5555008153571132e-06, 'epoch': 8.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.33065176010131836, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 0.9921, 'eval_samples_per_second': 878.905, 'eval_steps_per_second': 55.436, 'epoch': 8.01}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:13:16,976\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.34692466259002686, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 0.9923, 'eval_samples_per_second': 878.777, 'eval_steps_per_second': 55.427, 'epoch': 9.0}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'loss': 0.0547, 'learning_rate': 7.752576179103561e-07, 'epoch': 9.01}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.3511528968811035, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 1.0097, 'eval_samples_per_second': 863.616, 'eval_steps_per_second': 54.471, 'epoch': 9.01}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:14:16,986\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'eval_loss': 0.36141160130500793, 'eval_automl_metric': 0.06192660550458717, 'eval_runtime': 0.9929, 'eval_samples_per_second': 878.273, 'eval_steps_per_second': 55.396, 'epoch': 10.0}\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m {'train_runtime': 482.4828, 'train_samples_per_second': 207.261, 'train_steps_per_second': 12.954, 'train_loss': 0.1848659490966797, 'epoch': 10.0}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m Num examples = 872\n",+ "\u001B[2m\u001B[36m(train pid=86211)\u001B[0m Batch size = 16\n",+ "2022-03-19 14:14:22,379\tINFO tune.py:639 -- Total run time: 875.36 seconds (500.34 seconds for the tuning loop).\n",+ "[flaml.automl: 03-19 14:14:25] {2837} INFO - selected model: None\n",+ "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ " warnings.warn(\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'loss': 0.528, 'learning_rate': 8.898933352349567e-06, 'epoch': 1.0}\n",+ "{'loss': 0.2278, 'learning_rate': 1.3880017803076292e-05, 'epoch': 2.0}\n",+ "{'loss': 0.1455, 'learning_rate': 1.1410179501562432e-05, 'epoch': 3.0}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[flaml.automl: 03-19 14:17:03] {2947} INFO - retrain transformer for 157.7s\n",+ "[flaml.automl: 03-19 14:17:03] {2954} INFO - retrained model: None\n",+ "[flaml.automl: 03-19 14:17:03] {2283} INFO - fit succeeded\n",+ "[flaml.automl: 03-19 14:17:03] {2284} INFO - Time taken to find the best model: 370.53120946884155\n",+ "[flaml.automl: 03-19 14:17:03] {2295} WARNING - Time taken to find the best model is 74% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'train_runtime': 146.8805, 'train_samples_per_second': 519.019, 'train_steps_per_second': 32.441, 'train_loss': 0.30043953600021217, 'epoch': 3.0}\n"+ ]+ }+ ],+ "source": [+ "'''The main flaml automl API'''\n",+ "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Best model and metric"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Best hyperparmeter config: {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}\n",+ "Best accuracy on validation data: 0.9404\n",+ "Training duration of best run: 157.7 s\n"+ ]+ }+ ],+ "source": [+ "'''retrieve best config and best learner'''\n",+ "print('Best hyperparmeter config:', automl.best_config)\n",+ "print('Best accuracy on validation data: {0:.4g}'.format(1-automl.best_loss))\n",+ "print('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ " warnings.warn(\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'loss': 0.528, 'learning_rate': 8.898933352349567e-06, 'epoch': 1.0}\n",+ "{'eval_loss': 0.2549280524253845, 'eval_automl_metric': 0.08600917431192656, 'eval_runtime': 1.0003, 'eval_samples_per_second': 871.751, 'eval_steps_per_second': 54.984, 'epoch': 1.0}\n",+ "{'loss': 0.2278, 'learning_rate': 1.3880017803076292e-05, 'epoch': 2.0}\n",+ "{'eval_loss': 0.24966619908809662, 'eval_automl_metric': 0.06766055045871555, 'eval_runtime': 1.0201, 'eval_samples_per_second': 854.778, 'eval_steps_per_second': 53.914, 'epoch': 2.0}\n",+ "{'loss': 0.1455, 'learning_rate': 1.1410179501562432e-05, 'epoch': 3.0}\n",+ "{'eval_loss': 0.23046882450580597, 'eval_automl_metric': 0.059633027522935755, 'eval_runtime': 1.0097, 'eval_samples_per_second': 863.6, 'eval_steps_per_second': 54.47, 'epoch': 3.0}\n",+ "{'eval_loss': 0.23046882450580597, 'eval_automl_metric': 0.059633027522935755, 'eval_runtime': 0.9726, 'eval_samples_per_second': 896.568, 'eval_steps_per_second': 56.55, 'epoch': 3.0}\n",+ "{'train_runtime': 146.7879, 'train_samples_per_second': 519.346, 'train_steps_per_second': 32.462, 'train_loss': 0.30043953600021217, 'epoch': 3.0}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Using amp half precision backend\n",+ "***** Running Prediction *****\n",+ " Num examples = 872\n",+ " Batch size = 64\n"+ ]+ }+ ],+ "source": [+ "import pickle\n",+ "automl.pickle(\"automl.pkl\")\n",+ "\n",+ "with open(\"automl.pkl\", \"rb\") as f:\n",+ " automl = pickle.load(f)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Using amp half precision backend\n",+ "***** Running Prediction *****\n",+ " Num examples = 1821\n",+ " Batch size = 4\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Predicted labels [0 0 1 ... 1 1 1]\n"+ ]+ }+ ],+ "source": [+ "'''compute predictions of testing dataset''' \n",+ "y_pred = automl.predict(X_test, **{\"hf_args\": {\"per_device_eval_batch_size\": 1}})\n",+ "print('Predicted labels', y_pred)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Log history"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.4211966209891772e-05, 'num_train_epochs': 0.22667066737595704, 'per_device_train_batch_size': 4, 'warmup_ratio': 0.0638407085008166, 'weight_decay': 0.24365576482793252, 'adam_epsilon': 1.2017005181798623e-08, 'seed': 44, 'global_max_steps': 567, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 6.105850155006292e-06, 'num_train_epochs': 0.5203392488419986, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.02982185620209511, 'weight_decay': 0.06967105125024514, 'adam_epsilon': 2.8808753274240916e-07, 'seed': 42, 'global_max_steps': 163, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.029571768192494e-05, 'num_train_epochs': 0.23081935882872334, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.05927331754619176, 'weight_decay': 0.06990187648448716, 'adam_epsilon': 2.6275569358808137e-08, 'seed': 41, 'global_max_steps': 145, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00010208514406839324, 'num_train_epochs': 0.24916036398153807, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.28313267937361647, 'adam_epsilon': 1.5042842082223077e-08, 'seed': 44, 'global_max_steps': 78, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00010969207855501012, 'num_train_epochs': 0.5341197058945968, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.00727467370977386, 'weight_decay': 0.2998042286577564, 'adam_epsilon': 2.5359832189101376e-08, 'seed': 44, 'global_max_steps': 168, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00011786584823407102, 'num_train_epochs': 1.1449809097488275, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.015398649752204965, 'weight_decay': 0.3, 'adam_epsilon': 4.275263179285732e-08, 'seed': 43, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00011786584823407102, 'num_train_epochs': 1.1449809097488275, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.015398649752204965, 'weight_decay': 0.3, 'adam_epsilon': 4.275263179285732e-08, 'seed': 43, 'global_max_steps': 313, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 3.59933640287326e-05, 'num_train_epochs': 1.7937390219164033, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.016745580745240057, 'weight_decay': 0.2892279950527897, 'adam_epsilon': 6.514301011066509e-08, 'seed': 44, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 3.59933640287326e-05, 'num_train_epochs': 1.7937390219164033, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.016745580745240057, 'weight_decay': 0.2892279950527897, 'adam_epsilon': 6.514301011066509e-08, 'seed': 44, 'global_max_steps': 313, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.0001178658482340711, 'num_train_epochs': 1.1449809097488262, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.015398649752204969, 'weight_decay': 0.3, 'adam_epsilon': 4.275263179285729e-08, 'seed': 43, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 1878, 'learner': 'transformer'}}\n"+ ]+ }+ ],+ "source": [+ "from flaml.data import get_output_from_log\n",+ "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n",+ " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n",+ "for config in config_history:\n",+ " print(config)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "10\n"+ ]+ },+ {+ "data": {+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAiUElEQVR4nO3de7xVdZ3/8ddbQqHxAgg1CnKxkNRqxHZaWXkphfxVojkN9qtxbCamKf1VTs7gdHP050TZ5WePcXKwn2M2maFjRGWRhVrjJTmEglAYoilHSxApUwKBz/yxvhsW23X2WQfOvp3zfj4e+8Fe3/Vde332Pof9Oet7W4oIzMzMau3V6gDMzKw9OUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMNsNkt4gaVWr4zBrJCcI6ziSHpb05lbGEBE/jYgpjXp9SdMk/UTS05LWSbpd0tsbdT6zIk4QZgUkDWnhuc8EbgCuBcYBLwY+CbxtN15Lkvz/3HaLf3FswJC0l6TZkh6U9KSkeZJG5fbfIOk3kn6X/jo/MrfvGklflnSzpGeAE9OVykclLUvHfFPSsFT/BElrc8f3WDft/wdJj0t6TNLfSApJLy14DwK+AFwSEV+JiN9FxPaIuD0i3pfqXCTpP3PHTEyv94K0fZukSyXdATwLXCCpq+Y8H5G0ID3fR9LnJD0i6beSrpQ0fA9/HDYAOEHYQHIeMAM4HjgYeAq4Irf/+8Bk4EXAz4Gv1xz/LuBSYD/gv1PZO4HpwCTglcBf1Tl/YV1J04HzgTcDLwVOqPMaU4BDgBvr1CnjPcAssvdyJTBF0uTc/ncB16Xnc4DDgKNSfGPJrlhskHOCsIHk/cDHImJtRGwGLgLOrP5lHRFXR8TTuX1/JumA3PHfjog70l/sf0xlX4qIxyJiA/Adsi/RnvRU953Af0TEioh4Np27Jwemfx8v95Z7dE0639aI+B3wbeAsgJQoXgYsSFcss4CPRMSGiHga+Bdg5h6e3wYAJwgbSCYA35K0UdJG4BfANuDFkoZImpOan34PPJyOGZ07/tGC1/xN7vmzwL51zt9T3YNrXrvoPFVPpn8PqlOnjNpzXEdKEGRXD/NTshoDvBBYkvvcfpDKbZBzgrCB5FHgLRExIvcYFhHdZF+Kp5E18xwATEzHKHd8o5Y2fpyss7nqkDp1V5G9j3fUqfMM2Zd61Z8W1Kl9L7cAYyQdRZYoqs1L64FNwJG5z+yAiKiXCG2QcIKwTjVU0rDc4wVkbe2XSpoAIGmMpNNS/f2AzWR/ob+QrBmlWeYB50g6XNILgU/0VDGy9ffPBz4h6RxJ+6fO99dLmpuq3Qu8UdL41ER2YW8BRMRzZCOjLgNGkSUMImI7cBXwRUkvApA0VtK03X2zNnA4QVinupnsL9/q4yLgcmAB8ENJTwN3A8em+tcCvwa6gZVpX1NExPeBLwG3Aqtz597cQ/0bgb8A3gs8BvwW+L9k/QhExC3AN4FlwBLguyVDuY7sCuqGiNiaK//Halyp+e1HZJ3lNsjJNwwyay5JhwP3A/vUfFGbtRVfQZg1gaTT03yDkcBngO84OVi7c4Iwa46/BZ4AHiQbWfV3rQ3HrHduYjIzs0K+gjAzs0IvaHUA/WX06NExceLEVodhZtZRlixZsj4iCidGNjRBpDVoLgeGAF+JiDk1+ycAV5PN2twAvDsi1qZ924DlqeojEVF3qeOJEyfS1dVVr4qZmdWQ9Oue9jUsQaTlkq8ATgbWAoslLYiIlblqnwOujYivSjoJ+DTZImMAmyLiqEbFZ2Zm9TWyD+IYYHVErImILcD1ZEsd5B0BLErPby3Yb2ZmLdLIBDGWXRcMW5vK8u4DzkjPTwf2k1RdzXKYpC5Jd0uaUXQCSbNSna5169b1Y+hmZtbqUUwfBY6XtJRsDf9usjHiABMiokK2yNr/k/SS2oMjYm5EVCKiMmaMF580M+tPjeyk7mbXVSvHpbIdIuIx0hWEpH2Bd0TExrSvO/27RtJtwFSySUZmZtYEjUwQi4HJkiaRJYaZZFcDO0gaDWxIK0peSDaiibQcwbMRsTnVOQ74bANjNTPrGPOXdnPZwlU8tnETB48YzgXTpjBjam0L/p5rWBNTWmfmXGAh2Y1b5kXECkkXS6oOWT0BWCXpAbIbs1+ayg8HuiTdR9Z5Padm9JOZ2aA0f2k3F960nO6Nmwige+MmLrxpOfOXdvd6bF8NmKU2KpVKeB6EmQ10x81ZRPfGTc8rHztiOHfMPqnPrydpServfZ4BM5PazAauZjWpdILHCpJDvfI90epRTGZmdTWzSaUTHDxieJ/K94SvIMysrV22cBWbntu2S9mm57bxDzcu4xv3PNKiqFpn2NC92EuwPdc7MHzoEC6Y1v83AXSCMLO21lPTyZZt25scSXsYve8+ADy6YRNbtm1nbAOb3JwgzKytHTxieI+dst/829e2IKLBw30QZtbWLpg2heFDh+xS1qgmFduVryDMGsQjb/pH9TP7hxuXNbxJpdM0+nfMCcKsAaojb6qdq9WRN4C/2HbDjKljd3RIu1kp04zfMScIswbwyJv+t/Lx33PEQfu3Ooy20dPv2GULV/VbgnAfhFkDeORN/zvioP057ShffVU1Y8KcryDMGsAjb6zRevod688Jc76CMGsAj7yxRmvG75ivIMwawCNvrNGqv0sexWTWgTzyxhptxtSxDf2jw01MZmZWyFcQbc6TrcysVZwg2pgnW5lZKzlBtLFOmmy1/g+bd6wuufeQvThk1PAdq04OZp7cZZ3MCaKNdcpkq/V/2MxD65/ZsT79lm3beWj9MwCDPkl4cpd1MieINtYpk62Om7Nol5uXQHYzkz8+t72t4jSzvvEopjbWKZOtmnmPXDNrHieINjZj6lg+fcYr2HtI9mMaO2I4nz7jFW3XQd3Me+SaWfM0NEFImi5plaTVkmYX7J8g6ceSlkm6TdK43L6zJf0qPc5uZJztbMbUsUwdP4JjJ43ijtkntV1ygM650jGzvmlYH4SkIcAVwMnAWmCxpAURsTJX7XPAtRHxVUknAZ8G3iNpFPApoAIEsCQd+1R/x+l5BnuuGVP+zaz5GtlJfQywOiLWAEi6HjgNyCeII4Dz0/Nbgfnp+TTglojYkI69BZgOfKM/A/Q8g/7T6Cn/ZtZ8jUwQY4FHc9trgWNr6twHnAFcDpwO7CfpwB6Ofd63j6RZwCyA8ePH9znATpln4LH0ZtYKre6k/ihwvKSlwPFAN7Ct/iE7RcTciKhERGXMmDF9PnmnzDPwWHoza4VGXkF0A4fktselsh0i4jGyKwgk7Qu8IyI2SuoGTqg59rb+DrBT5hmYmbVCI68gFgOTJU2StDcwE1iQryBptKRqDBcCV6fnC4FTJI2UNBI4JZX1K4++MTPrWcMSRERsBc4l+2L/BTAvIlZIuljS21O1E4BVkh4AXgxcmo7dAFxClmQWAxdXO6z7U3WewdgRwxHtO8/AzKwVFBG91+oAlUolurq6Wh2GmVlHkbQkIipF+1rdSW1mZm3Ki/XV8MQ5M7OME0SOJ86Zme3kJqacnibOXbZwVYsiMjNrHSeIHC9bbWa2kxNEjpetNjPbyQkixxPn+s/8pd0cN2cRk2Z/j+PmLGL+0u7eDzKztuJO6hwvW90/3NlvNjA4QdTwstV7rl5nvz9bs87hJibrd+7sNxsYnCCs37mz32xgcIKwfufOfrOBwX0Q1u/c2W82MDhBWEO4s9+s87mJyczMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKyQE4SZmRXqNUFIOrAZgZiZWXspcwVxt6QbJJ0qSQ2PyMzM2kKZBHEYMBd4D/ArSf8i6bAyLy5puqRVklZLml2wf7ykWyUtlbRM0qmpfKKkTZLuTY8r+/KmzMxsz/U6kzoiArgFuEXSicB/Ah+QdB8wOyLuKjpO0hDgCuBkYC2wWNKCiFiZq/ZxYF5EfFnSEcDNwMS078GIOGr33paZme2pXhNE6oN4N9kVxG+B84AFwFHADcCkHg49BlgdEWvS61wPnAbkE0QA+6fnBwCP9fkdmJlZQ5RpYrqL7Et8RkT8r4i4KSK2RkQXUK/pZyzwaG57bSrLuwh4t6S1ZFcP5+X2TUpNT7dLekPRCSTNktQlqWvdunUl3oqZmZVVZrG+KamZ6Xki4jN7eP6zgGsi4vOSXgt8TdLLgceB8RHxpKRXAfMlHRkRv685/1yy/hEqlUphjGZmtnvKXEH8UNKI6oakkZIWljiuGzgktz0uleX9NTAPIPVlDANGR8TmiHgylS8BHiTrLDczsyYpkyDGRMTG6kZEPAW8qMRxi4HJkiZJ2huYSdZ3kfcI8CYASYeTJYh1ksakTm4kHQpMBtaUOKeZmfWTMk1M2ySNj4hHACRNIOtcrisitko6F1gIDAGujogVki4GuiJiAfD3wFWSPpJe868iIiS9EbhY0nPAduD9EbFht96hmZntFvXQvbCzgjSdrJ3/dkDAG4BZEVGmmalpKpVKdHV1tToMM7OOImlJRFSK9pWZB/EDSUcDr0lFH46I9f0ZoJmZtZ+ytxzdBjxB1kdwhCQi4ieNC8vMzFqtzES5vwE+RDYK6V6yK4m7gJMaGpmZmbVUmVFMHwJeDfw6Ik4EpgIbGxmUmZm1XpkE8ceI+COApH0i4pfAlMaGZWZmrVamD2Jtmig3n2zBvqeAXzcyKDMza70yo5hOT08vknQr2aJ6P2hoVGZm1nJ1E0SazbwiIl4GEBG3NyUqMzNrubp9EBGxDVglaXyT4jEzszZRpg9iJLBC0j3AM9XCiHh7w6IyM7OWK5MgPtHwKMzMrO2U6aR2v4OZ2SBUZib10+xcvXVvYCjwTETs3/NRZmbW6cpcQexXfS5JZPeVfk3PR5iZ2UBQZib1DpGZD0xrTDhmZtYuyjQxnZHb3AuoAH9sWERmZtYWyoxielvu+VbgYbJmJjMzG8DK9EGc04xAzMysvfTaByHpq2mxvur2SElXNzQqMzNruTKd1K+MiI3VjYh4iuyeEGZmNoCVSRB7SRpZ3ZA0ivK3KjUzsw5VJkF8HrhL0iWSLgHuBD5b5sUlTZe0StJqSbML9o+XdKukpZKWSTo1t+/CdNwqSR5Wa2bWZGU6qa+V1MXOe1CfERErezsuLRV+BXAysBZYLGlBzbEfB+ZFxJclHQHcDExMz2cCRwIHAz+SdFhaXdbMzJqgTCf1a4BHI+JfI+Jfye4wd2yJ1z4GWB0RayJiC3A9zx8eG0B1yY4DgMfS89OA6yNic0Q8BKxOr2dmZk1Sponpy8Afctt/SGW9GQs8mttem8ryLgLeLWkt2dXDeX04FkmzJHVJ6lq3bl2JkMzMrKwyCUIRUV2sj4jYTv91Up8FXBMR44BTga9JKr38R0TMjYhKRFTGjBnTTyGZmRmUSxBrJP0fSUPT40PAmhLHdQOH5LbHpbK8vwbmAUTEXcAwYHTJY83MrIHKJIj3A68j+4JeCxwLvK/EcYuByZImSdqbrNN5QU2dR4A3AUg6nCxBrEv1ZkraR9IkYDJwT4lzmplZPykziukJsi93ACQNB94K3NDLcVslnQssBIYAV0fECkkXA10RsQD4e+AqSR8h67D+q9SctULSPGAl2fpPH/QIJjOz5lKue6HnStmQ1WlkfQYnA/8dEWc2OLY+qVQq0dXV1eowzMw6iqQlEVEp2lf3CkLS8cC7yDqQ7wGOAw6NiGf7PUozM2srPSaINPT0EbIhrR+NiKclPeTkYGY2ONTrpL6RbBbzXwBvk/Qn7Lw3tZmZDXA9JoiI+DAwiWwtphOAVcAYSe+UtG9TojMzs5apO8w13YP61oiYRZYsziJbBuPhJsRmZmYtVHpGdEQ8B3wX+G4a6mpmZgNY6WUt8iJiU38HYmZm7WW3EoSZmQ18ThBmZlao1z4ISYcBFwAT8vUj4qQeDzIzs45XppP6BuBK4CrA6yGZmQ0SZRLE1ogoc4MgMzMbQMr0QXxH0gckHSRpVPXR8MjMzKylylxBnJ3+vSBXFsCh/R+OmZm1izL3g5jUjEDMzKy9lBnFNBT4O+CNqeg24N/TzGozMxugyjQxfRkYCvxb2n5PKvubRgVlZmatVyZBvDoi/iy3vUjSfY0KyMzM2kOZUUzbJL2kuiHpUDwfwsxswCtzBXEBcKukNYDIZlSf09CozMys5cqMYvqxpMnAlFS0KiI2NzYsMzNrtXr3pD4pIhZJOqNm10slERE3NTg2MzNroXpXEMcDi4C3FewLoNcEIWk6cDkwBPhKRMyp2f9F4MS0+ULgRRExIu3bBixP+x6JiLf3dj4zM+s/PSaIiPhUenpxRDyU3yep18lzkoYAVwAnA2uBxZIWRMTK3Dk+kqt/HjA19xKbIuKoMm/CzMz6X5lRTP9VUHZjieOOAVZHxJqI2AJcT3Y/656cBXyjxOuamVkT1OuDeBlwJHBATT/E/sCwEq89Fng0t70WOLaHc00AJpE1aVUNk9QFbAXmRMT8guNmAbMAxo8fXyIkMzMrq14fxBTgrcAIdu2HeBp4Xz/HMRO4MSLy8ysmRER3mnexSNLyiHgwf1BEzAXmAlQqlejnmMzMBrV6fRDfBr4t6bURcdduvHY3cEhue1wqKzIT+GDN+bvTv2sk3UbWP/Hg8w81M7NGKDNRbqmkD5I1N+1oWoqI9/Zy3GJgcurQ7iZLAu+qrZSaskYCd+XKRgLPRsRmSaOB44DPlojVzMz6SZlO6q8BfwpMA24nuxJ4ureDImIrcC6wEPgFMC8iVki6WFJ+yOpM4PqIyDcRHQ50pTWfbiXrg1iJmZk1jXb9Xi6oIC2NiKmSlkXEK9Py3z+NiNc0J8RyKpVKdHV1tToMM7OOImlJRFSK9pW5gqje92GjpJcDBwAv6q/gzMysPZXpg5ib+gQ+ASwA9gU+2dCozMys5cos1veV9PR2fB9qM7NBo95EufPrHRgRX+j/cMzMrF3Uu4LYL/07BXg1WfMSZJPm7mlkUGZm1nr1Jsr9M4CknwBHR8TTafsi4HtNic7MzFqmzCimFwNbcttbUpmZmQ1gZUYxXQvcI+lbaXsGcE2jAjIzs/ZQZhTTpZK+D7whFZ0TEUsbG5aZmbVavVFM+0fE7yWNAh5Oj+q+URGxofHhmZlZq9S7griObLnvJWS3GK1S2vacCDOzAazeKKa3pn97vb2omZkNPPWamI6ud2BE/Lz/wzEzs3ZRr4np83X2BXBSP8diZmZtpF4T04nNDMTMzNpLmXkQpGW+j2DXO8pd26igzMys9XpNEJI+BZxAliBuBt4C/DfZBDozMxugyiy1cSbwJuA3EXEO8GdkNw0yM7MBrEyC2BQR24GtkvYHngAOaWxYZmbWamX6ILokjQCuIps09wfgrkYGZWZmrVdvHsQVwHUR8YFUdKWkHwD7R8SypkRnZmYtU+8K4gHgc5IOAuYB3/AifWZmg0ePfRARcXlEvBY4HngSuFrSLyV9StJhZV5c0nRJqyStljS7YP8XJd2bHg9I2pjbd7akX6XH2X1/a2ZmticUEb3XqlaWpgJXA6+MiCG91B1CdhVyMrAWWAycFREre6h/HjA1It6bVpDtAipks7aXAK+KiKd6Ol+lUomurq7S78XMzEDSkoioFO3rdRSTpBdIepukrwPfB1YBZ5Q47zHA6ohYExFbgOuB0+rUPwv4Rno+DbglIjakpHALML3EOc3MrJ/U66Q+mexL+1TgHrIv+FkR8UzJ1x4LPJrbXgsc28O5JgCTgEV1jh1bcNwsYBbA+PHjS4ZlZmZl1LuCuBC4Ezg8It4eEdf1ITn01UzgxojY1peDImJuRFQiojJmzJgGhWZmNjjVW6xvT1dr7WbXCXXjUlmRmcAHa449oebY2/YwHjMz64MyM6l312JgsqRJkvYmSwILaitJehkwkl0n3y0ETpE0UtJI4JRUZmZmTVJqNdfdERFbJZ1L9sU+BLg6IlZIuhjoiohqspgJXB+54VQRsUHSJWRJBuBi3wPbzKy5+jTMtZ15mKuZWd/t0TBXMzMbnJwgzMyskBOEmZkVcoIwM7NCThBmZlbICcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgzMyskBOEmZkVcoIwM7NCThBmZlbICcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgzMyskBOEmZkVcoIwM7NCThBmZlaooQlC0nRJqyStljS7hzrvlLRS0gpJ1+XKt0m6Nz0WNDJOMzN7vhc06oUlDQGuAE4G1gKLJS2IiJW5OpOBC4HjIuIpSS/KvcSmiDiqUfGZmVl9jbyCOAZYHRFrImILcD1wWk2d9wFXRMRTABHxRAPjMTOzPmhkghgLPJrbXpvK8g4DDpN0h6S7JU3P7RsmqSuVzyg6gaRZqU7XunXr+jV4M7PBrmFNTH04/2TgBGAc8BNJr4iIjcCEiOiWdCiwSNLyiHgwf3BEzAXmAlQqlWhq5GZmA1wjryC6gUNy2+NSWd5aYEFEPBcRDwEPkCUMIqI7/bsGuA2Y2sBYzcysRiMTxGJgsqRJkvYGZgK1o5Hmk109IGk0WZPTGkkjJe2TKz8OWImZmTVNw5qYImKrpHOBhcAQ4OqIWCHpYqArIhakfadIWglsAy6IiCclvQ74d0nbyZLYnPzoJzMzazxFDIym+0qlEl1dXa0Ow8yso0haEhGVon2eSW1mZoWcIMzMrJAThJmZFXKCMDOzQk4QZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFWr1/SBsD8xf2s1lC1fx2MZNHDxiOBdMm8KMqbX3ZDIz2z1OEB1q/tJuLrxpOZue2wZA98ZNXHjTcgAnCTPrF25i6lCXLVy1IzlUbXpuG5ctXNWiiMxsoHGC6FCPbdzUp3Izs75yguhQB48Y3qdyM7O+coLoUBdMm8LwoUN2KRs+dAgXTJvSoojMbKBxJ3WHqnZEexSTmTWKE0QHmzF1rBOCmTWMm5jMzKyQE4SZmRVygjAzs0JOEGZmVsgJwszMCikiWh1Dv5C0Dvh1iaqjgfUNDqeRHH9rOf7Wcvz9b0JEjCnaMWASRFmSuiKi0uo4dpfjby3H31qOv7ncxGRmZoWcIMzMrNBgTBBzWx3AHnL8reX4W8vxN9Gg64MwM7NyBuMVhJmZleAEYWZmhQZ8gpD0sKTlku6V1JXKRkm6RdKv0r8jWx1nlaSrJT0h6f5cWWG8ynxJ0mpJyyQd3brId8RaFP9FkrrTz+BeSafm9l2Y4l8laVprot4RyyGSbpW0UtIKSR9K5R3x+deJvyM+/xTPMEn3SLovvYd/TuWTJP0sxfpNSXun8n3S9uq0f2Kbxn+NpIdyP4OjUnlb/Q49T0QM6AfwMDC6puyzwOz0fDbwmVbHmYvtjcDRwP29xQucCnwfEPAa4GdtGv9FwEcL6h4B3AfsA0wCHgSGtDD2g4Cj0/P9gAdSjB3x+deJvyM+/xSTgH3T86HAz9JnOw+YmcqvBP4uPf8AcGV6PhP4ZpvGfw1wZkH9tvodqn0M+CuIHpwGfDU9/yowo3Wh7CoifgJsqCnuKd7TgGsjczcwQtJBTQm0Bz3E35PTgOsjYnNEPASsBo5pWHC9iIjHI+Ln6fnTwC+AsXTI518n/p601ecPkD7LP6TNoekRwEnAjam89mdQ/dncCLxJkpoT7fPVib8nbfU7VGswJIgAfihpiaRZqezFEfF4ev4b4MWtCa20nuIdCzyaq7eW+l8IrXRuuoS+Otek17bxp6aKqWR/AXbc518TP3TQ5y9piKR7gSeAW8iubDZGxNZUJR/njveQ9v8OOLCpAdeojT8iqj+DS9PP4IuS9kllbfkzqBoMCeL1EXE08Bbgg5LemN8Z2XVex4z17bR4ky8DLwGOAh4HPt/SaHohaV/gv4APR8Tv8/s64fMviL+jPv+I2BYRRwHjyK5oXtbaiPqmNn5JLwcuJHsfrwZGAf/YugjLG/AJIiK6079PAN8i+4X7bfUyLv37ROsiLKWneLuBQ3L1xqWythIRv03/abYDV7GzGaPt4pc0lOzL9esRcVMq7pjPvyj+Tvr88yJiI3Ar8FqyppfqLZLzce54D2n/AcCTzY20WC7+6an5LyJiM/AfdMjPYEAnCEl/Imm/6nPgFOB+YAFwdqp2NvDt1kRYWk/xLgD+Mo2EeA3wu1xTSNuoaVM9nexnAFn8M9NIlEnAZOCeZsdXldqu/z/wi4j4Qm5XR3z+PcXfKZ8/gKQxkkak58OBk8n6Um4FzkzVan8G1Z/NmcCidJXXEj3E/8vcHxgi6z/J/wza5nfoeVrdS97IB3Ao2SiN+4AVwMdS+YHAj4FfAT8CRrU61lzM3yBrBniOrD3yr3uKl2zkwxVkbbTLgUqbxv+1FN8ysv8QB+XqfyzFvwp4S4tjfz1Z89Ey4N70OLVTPv868XfE55/ieSWwNMV6P/DJVH4oWfJaDdwA7JPKh6Xt1Wn/oW0a/6L0M7gf+E92jnRqq9+h2oeX2jAzs0IDuonJzMx2nxOEmZkVcoIwM7NCThBmZlbICcLMzAo5QVhHSMsTfDi3vVDSV3Lbn5d0fp3jr5F0Znp+m6Tn3The0lBJc5St2vpzSXdJekva97Ck0bsR947z9rD/irS650pJm3KrfZ4p6ebqmPr+JOkgSd+ts39vST/JTUyzQcoJwjrFHcDrACTtBYwGjsztfx1w5x6e4xKyFVFfHtnyLDPIVkVtmIj4YGTLMpwKPBgRR6XHjRFxamSzcfvb+WQzqnuKaQvZvI+/aMC5rYM4QVinuJNsyQXIEsP9wNOSRqaFzw4Hfi7pk5IWS7pf0tyyK3tKeiHwPuC8yJZDILIlKuYV1D0/vf79NVc1f5kWY7tP0tcKjrskXVEMKRnTw5JGS5oo6Zfp2AckfV3SmyXdka52jkn1/0TZYnz3SFoq6bQeXvodwA/SMUem+vem2CenOvOB/10mThu4fAlpHSEiHpO0VdJ4squFu8hWvXwt2QqeyyNii6R/jYiLAdKX9FuB75Q4xUuBR6Jmcb5akl4FnAMcSzYL9meSbge2AB8HXhcR6yWNqjnuMrKrkXNi92anvhT4c+C9wGLgXWQzp98O/BPZ1c7HyJaaeG9qmrpH0o8i4plcHJOAp6pJEHg/cHlEfF3ZTXiqyet+soXlbBDzFYR1kjvJkkM1QdyV274j1TlR2Z3FlpPdQ+DIohfaA68HvhURz0S27v9NwBvSuW6IiPUAEZG/J8YngAMi4v27mRwAHoqI5ZEtuLcC+HF6reXAxFTnFGC2sqWmbyNbhmJ8zescBKzLbd8F/JOkfwQmRMSmFP82YIvSWmY2ODlBWCep9kO8guwv3LvJriBeB9wpaRjwb2R37noFWTv7sJKvvRoYL2n/fo86+4v/VbVXFX20Ofd8e257OztbAgS8I9ePMT4iflHzOpvIfSYRcR3ZVcgm4GZJJ+Xq7gP8cQ9itg7nBGGd5E6yJqMNkS1fvQEYQZYk7mTnF996ZfdE6HH0UK2IeJZsJdTLtfN+x2Mk/XlN1Z8CMyS9UNkKwaenskXAn0s6MB2bTwY/AOYA32vwX+QLgfOq/S6SphbUeYCdVxxIOhRYExFfIlsh9ZWp/EBgfUQ818B4rc05QVgnWU42eunumrLfRcT6NOLnKrKri4Vkf7n3xcfJml9WSrof+C5Qe8Ogn5PdX/gesru1fSUilkbECuBS4HZJ9wFfqDnuhhTbgrQMdCNcQnaLy2WSVqTtXaT+iAclvTQVvRO4PzVLvRy4NpWfCHyvQXFah/BqrmaDjKTTgVdFxMfr1LkJmB0RDzQvMms3HsVkNshExLeqTWFFUhPbfCcH8xWEmZkVch+EmZkVcoIwM7NCThBmZlbICcLMzAo5QZiZWaH/AVC/n89BGYiaAAAAAElFTkSuQmCC",+ "text/plain": [+ "<Figure size 432x288 with 1 Axes>"+ ]+ },+ "metadata": {+ "needs_background": "light"+ },+ "output_type": "display_data"+ }+ ],+ "source": [+ "import matplotlib.pyplot as plt\n",+ "import numpy as np\n",+ "\n",+ "plt.title('Learning Curve')\n",+ "plt.xlabel('Wall Clock Time (s)')\n",+ "plt.ylabel('Validation Accuracy')\n",+ "print(len(valid_loss_history))\n",+ "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n",+ "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 3. Other Tasks"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Besides sequence classification, FLAML currently also supports four other tasks (more tasks are to be supported, which can be found on FLAML's documentation website https://microsoft.github.io/FLAML/docs/Examples/AutoML-NLP):\n",+ "\n",+ "- sequence regression: predicting a float number from the input sequence, e.g., predicting the rating of a hotel review based on the text content;\n",+ "- token classification: predicting the label of each token in a sequence, e.g., named entity recognition;\n",+ "- multiple choice: predicting the best second half of a sentence that comes next to the first part of a sentence based on common sensen reasoning. An example is seen below;\n",+ "- (abstractive) summarization: generating the textual summarization of an input paragraph;\n",+ "\n",+ "For each task, you only have to change the \"Load data and preprocess\" with the corresponding data loading process. For example:"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 3.1 Multiple Choice Example"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Multiple choice is a task of predicting the best second half of a sentence that follows the first half based on common sense reasoning. An example of multiple-choice classification problem is:\n",+ "\n",+ "On stage, a woman takes a seat at the piano. She\n",+ "a) sits on a bench as her sister plays with the doll.\n",+ "b) smiles with someone as the music plays.\n",+ "c) is in the crowd, watching the dancers.\n",+ "d) *nervously sets her fingers on the keys*."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "No config specified, defaulting to: swag/regular\n",+ "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n",+ "No config specified, defaulting to: swag/regular\n",+ "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n",+ "No config specified, defaulting to: swag/regular\n",+ "Reusing dataset swag (/home/xliu127/.cache/huggingface/datasets/swag/regular/0.0.0/9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c)\n"+ ]+ }+ ],+ "source": [+ "from datasets import load_dataset\n",+ "\n",+ "train_dataset = load_dataset(\"swag\", split=\"train\").to_pandas().iloc[:10000]\n",+ "dev_dataset = load_dataset(\"swag\", split=\"validation\").to_pandas().iloc[:10000]\n",+ "test_dataset = load_dataset(\"swag\", split=\"test\").to_pandas()\n",+ "\n",+ "custom_sent_keys = [\n",+ " \"sent1\",\n",+ " \"sent2\",\n",+ " \"ending0\",\n",+ " \"ending1\",\n",+ " \"ending2\",\n",+ " \"ending3\",\n",+ " \"gold-source\",\n",+ " \"video-id\",\n",+ " \"startphrase\",\n",+ " \"fold-ind\",\n",+ " ] # specify the column names of the input sentences\n",+ "label_key = \"label\" # specify the column name of the label\n",+ "\n",+ "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n",+ "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n",+ "X_test = test_dataset[custom_sent_keys]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "'Members of the procession walk down the street holding small horn brass instruments.'"+ ]+ },+ "execution_count": 7,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "train_dataset.iloc[0][\"sent1\"]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 13,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/html": [+ "== Status ==<br>Current time: 2022-03-19 14:39:29 (running for 00:08:29.94)<br>Memory usage on this node: 33.0/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/250.17 GiB heap, 0.0/111.21 GiB objects (0.0/1.0 accelerator_type:V100)<br>Current best trial: de45e672 with val_loss=0.18300000000000005 and parameters={'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}<br>Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-03-19_14-30-59<br>Number of trials: 10/1000000 (10 TERMINATED)<br><br>"+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m {'eval_loss': 0.6315866112709045, 'eval_automl_metric': 0.18779999999999997, 'eval_runtime': 15.4883, 'eval_samples_per_second': 645.648, 'eval_steps_per_second': 40.353, 'epoch': 1.66}\n",+ "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m {'train_runtime': 190.7625, 'train_samples_per_second': 87.254, 'train_steps_per_second': 10.909, 'train_loss': 0.5091343906738046, 'epoch': 1.66}\n",+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.2585, 'eval_samples_per_second': 655.374, 'eval_steps_per_second': 40.961, 'epoch': 2.87}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source. If ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n",+ "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m Num examples = 10000\n",+ "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m Batch size = 16\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.1369, 'eval_samples_per_second': 660.639, 'eval_steps_per_second': 41.29, 'epoch': 2.87}\n",+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m {'train_runtime': 546.3809, 'train_samples_per_second': 156.658, 'train_steps_per_second': 39.165, 'train_loss': 0.5030154804349909, 'epoch': 2.87}\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'loss': 0.4854, 'learning_rate': 1.3592147782116173e-06, 'epoch': 2.0}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3. If fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3 are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n",+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m Num examples = 10000\n",+ "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m Batch size = 16\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.49709731340408325, 'eval_automl_metric': 0.17600000000000005, 'eval_runtime': 15.4983, 'eval_samples_per_second': 645.232, 'eval_steps_per_second': 40.327, 'epoch': 2.0}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:41:56,719\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.5254333019256592, 'eval_automl_metric': 0.17800000000000005, 'eval_runtime': 15.45, 'eval_samples_per_second': 647.251, 'eval_steps_per_second': 40.453, 'epoch': 3.0}\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'loss': 0.3989, 'learning_rate': 3.8051750127352887e-07, 'epoch': 3.0}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:42:56,729\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.5254867076873779, 'eval_automl_metric': 0.17789999999999995, 'eval_runtime': 15.424, 'eval_samples_per_second': 648.341, 'eval_steps_per_second': 40.521, 'epoch': 3.0}\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.5332269072532654, 'eval_automl_metric': 0.17830000000000001, 'eval_runtime': 15.4452, 'eval_samples_per_second': 647.45, 'eval_steps_per_second': 40.466, 'epoch': 3.39}\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'train_runtime': 382.2827, 'train_samples_per_second': 88.597, 'train_steps_per_second': 11.076, 'train_loss': 0.5299136270370808, 'epoch': 3.39}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:43:56,739\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind. If ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m Num examples = 10000\n",+ "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m Batch size = 16\n",+ "2022-03-19 14:44:14,271\tINFO tune.py:639 -- Total run time: 795.18 seconds (504.18 seconds for the tuning loop).\n",+ "[flaml.automl: 03-19 14:44:19] {2837} INFO - selected model: None\n",+ "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ " warnings.warn(\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'loss': 0.6603, 'learning_rate': 4.631567529441369e-06, 'epoch': 1.0}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[flaml.automl: 03-19 14:46:08] {2947} INFO - retrain transformer for 109.2s\n",+ "[flaml.automl: 03-19 14:46:08] {2954} INFO - retrained model: None\n",+ "[flaml.automl: 03-19 14:46:08] {2283} INFO - fit succeeded\n",+ "[flaml.automl: 03-19 14:46:08] {2284} INFO - Time taken to find the best model: 319.927033662796\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'train_runtime': 96.899, 'train_samples_per_second': 245.031, 'train_steps_per_second': 30.63, 'train_loss': 0.6602518278346073, 'epoch': 1.0}\n"+ ]+ }+ ],+ "source": [+ "''' import AutoML class from flaml package '''\n",+ "from flaml import AutoML\n",+ "automl = AutoML()\n",+ "\n",+ "if not ray.is_initialized():\n",+ " ray.init()\n",+ "\n",+ "automl_settings = {\n",+ " \"time_budget\": 500, # setting the time budget\n",+ " \"task\": \"multichoice-classification\", # setting the task as multiplechoice-classification\n",+ " \"hf_args\": {\n",+ " \"output_dir\": \"data/output/\", # setting the output directory\n",+ " \"ckpt_per_epoch\": 1, # setting the number of checkoints per epoch\n",+ " \"per_device_eval_batch_size\": 16, # the batch size for validation (inference)\n",+ " },\n",+ " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n",+ " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n",+ " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n",+ " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n",+ " \"n_concurrent_trials\": 4\n",+ "}\n",+ "\n",+ "'''The main flaml automl API'''\n",+ "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 14,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.00021956991427751982, 'num_train_epochs': 0.3549576494055084, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.07425273520338253, 'weight_decay': 0.03879221030529465, 'adam_epsilon': 3.7880482987985576e-08, 'seed': 43, 'global_max_steps': 444, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 0.00021956991427751982, 'num_train_epochs': 0.3549576494055084, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.07425273520338253, 'weight_decay': 0.03879221030529465, 'adam_epsilon': 3.7880482987985576e-08, 'seed': 43, 'global_max_steps': 444, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.3241899893349513e-06, 'num_train_epochs': 0.4379128434860086, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.257055208282222, 'weight_decay': 0.012652183020312091, 'adam_epsilon': 1.0189125195705357e-07, 'seed': 43, 'global_max_steps': 274, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 0.0002562922748967212, 'num_train_epochs': 0.1802995999606059, 'per_device_train_batch_size': 4, 'warmup_ratio': 0.1809477882684876, 'weight_decay': 0.10305626005953175, 'adam_epsilon': 5.536776887412208e-08, 'seed': 42, 'global_max_steps': 451, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 313, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 1251, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 1251, 'learner': 'transformer'}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 9.306519250357542e-06, 'num_train_epochs': 0.4664878701006166, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 5.931759315303309e-07, 'seed': 43, 'global_max_steps': 147, 'learner': 'transformer'}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 6.104513714676502e-06, 'num_train_epochs': 2.3743291981165893, 'per_device_train_batch_size': 8, 'warmup_ratio': 0.23610846764298543, 'weight_decay': 0.20205904544254147, 'adam_epsilon': 5.752964074991208e-08, 'seed': 41, 'global_max_steps': 1251, 'learner': 'transformer'}}\n",+ "6\n"+ ]+ },+ {+ "data": {+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAd8klEQVR4nO3df9wVZZ3/8dfbWxQqFY27VhGEEinNEiN/9GMzy8XcFDIzbLc126S2NMuiZEtjbd0yV3vUY9l8oF/zxzdDZc3IKHLTrBQF/IlgGCkqaIo/UFIUwc/+MdfB8XjuwwBnzrnve97Px+M8mLnmOjOfM8x9Pmeua+YaRQRmZlZdW3U6ADMz6ywnAjOzinMiMDOrOCcCM7OKcyIwM6s4JwIzs4pzIjBrQtJ7JC3pdBxmZXIisF5L0jJJH+hkDBHx+4gYXdb6JY2T9DtJqyWtlHS9pCPK2p5ZI04EVmmSujq47aOAK4CLgV2B1wOnAYdvxrokyX/Ptll84FifI2krSadI+rOkxyVdLmmn3PIrJP1F0lPp1/ZeuWUXSvqhpNmSngHel848viLpzvSeyyQNTPUPkrQ89/4e66blX5X0sKSHJH1aUkjavcFnEHAO8K2IOD8inoqIFyPi+og4PtWZKun/594zIq1v6zT/W0lnSLoBeBaYLGlB3Xa+JGlWmt5W0n9KekDSI5LOlTRoC/87rB9wIrC+6ERgAvBeYBfgSWBabvkvgVHA64BbgR/Xvf/jwBnAdsAfUtnRwKHASOCtwCebbL9hXUmHAicDHwB2Bw5qso7RwDBgZpM6RXwCmET2Wc4FRksalVv+ceDSNP0dYA9gnxTfULIzEKs4JwLriz4LfD0ilkfE88BU4KjaL+WIuCAiVueWvU3SDrn3/ywibki/wJ9LZT+IiIci4gng52Rflj3pqe7RwI8iYlFEPJu23ZPXpn8fLvaRe3Rh2t66iHgK+BlwDEBKCG8CZqUzkEnAlyLiiYhYDfwHMHELt2/9gBOB9UW7AT+VtErSKuBuYD3wekldkr6Tmo2eBpal9wzJvf/BBuv8S276WeA1TbbfU91d6tbdaDs1j6d/d25Sp4j6bVxKSgRkZwNXpaTUDbwKuCW3336Vyq3inAisL3oQ+GBEDM69BkbECrIvv/FkzTM7ACPSe5R7f1lD7j5M1ulbM6xJ3SVkn+MjTeo8Q/blXfM3DerUf5ZrgG5J+5AlhFqz0GPAGmCv3D7bISKaJTyrCCcC6+0GSBqYe21N1hZ+hqTdACR1Sxqf6m8HPE/2i/tVZM0f7XI5cJykN0t6FXBqTxUjG//9ZOBUScdJ2j51gr9b0vRU7XbgbyUNT01bUzYWQES8QHYl0lnATmSJgYh4ETgP+J6k1wFIGipp3OZ+WOs/nAist5tN9ku29poKfB+YBfxa0mrgJmD/VP9i4H5gBbA4LWuLiPgl8APgOmBpbtvP91B/JvAx4FPAQ8AjwL+TtfMTEdcAlwF3ArcAVxcM5VKyM6IrImJdrvxrtbhSs9n/knVaW8XJD6YxK4ekNwN3AdvWfSGb9So+IzBrIUkfTtfr7wicCfzcScB6OycCs9b6DPAo8GeyK5n+pbPhmG2cm4bMzCrOZwRmZhW3dacD2FRDhgyJESNGdDoMM7M+5ZZbbnksIhreQNjnEsGIESNYsGDBxiuamdkGku7vaZmbhszMKs6JwMys4pwIzMwqzonAzKzinAjMzCquz101ZGa2Oa66bQVnzVnCQ6vWsMvgQUweN5oJY4Z2OqxewYmgBXyAmfVuV922gilXLmTNC+sBWLFqDVOuXAjgv1WcCLaYDzCz3u+sOUs2/I3WrHlhPV+deSc/mfdAh6LadHvusj3fPHyvlq/XiWAL9ZcDzKw/W7FqTcPytetfbHMkvZMTwRZ6yAeYWa+3TddWDf8mhw4exGWfObADEW2aWvPzvPue4NeLHml587MTwRbaZfCghr82+soBZlYF9U24AIMGdDF5XO9/QFs7mp9LvXxU0qGSlkhaKumUBsuHS7pO0m2S7pR0WJnxlGHyuNEMGtD1srK+coCZVcWEMUP59pF7M3TwIET2Q+3bR+7dJ/rxemp+PmvOkpZto7QzAkldwDTgEGA5MF/SrIhYnKv2DeDyiPihpD3Jnk87oqyYylA7kL46807Wrn+Rob5qyKxXmjBmaJ/8u+yp+bmn8s1RZtPQfsDSiLgXQNIMYDzZA8VrAtg+Te9A9gDvPmfCmKEbOobdHGRmrdRT8/Mugwe1bBtlNg0NBR7MzS9PZXlTgX+UtJzsbODERiuSNEnSAkkLVq5cWUasZma9Ujuanzs9xMQxwIURsStwGHCJpFfEFBHTI2JsRIzt7m74XAUzs36pHf0bZTYNrQCG5eZ3TWV5/wwcChARcyUNBIaQPfzbzMwov3+jzDOC+cAoSSMlbQNMBGbV1XkAeD+ApDcDAwG3/ZiZtVFpiSAi1gEnAHOAu8muDlok6XRJR6RqXwaOl3QH8BPgkxERZcVkZmavVOoNZRExm6wTOF92Wm56MfCuMmMwazcPQmh9je8sNmshD0JofVGnrxoy61facReoWas5EZi1UDvuAjVrNScCsxbq6W7PVt4FatZqTgRmLeRBCK0vcmexWQvVOoR91ZD1JU4EZi3WV0e5tOpy05CZWcU5EZiZVZwTgZlZxTkRmJlVnBOBmVnFORGYmVWcE4GZWcU5EZiZVZwTgZlZxTkRmJlVnBOBmVnFORGYmVWcE4GZWcU5EZiZVZwTgZlZxTkRmJlVnBOBmVnFORGYmVWcE4GZWcWVmggkHSppiaSlkk5psPx7km5Pr3skrSozHjMze6XSHl4vqQuYBhwCLAfmS5oVEYtrdSLiS7n6JwJjyorHzMwaK/OMYD9gaUTcGxFrgRnA+Cb1jwF+UmI8ZmbWQJmJYCjwYG5+eSp7BUm7ASOBa3tYPknSAkkLVq5c2fJAzcyqrLd0Fk8EZkbE+kYLI2J6RIyNiLHd3d1tDs3MrH8rMxGsAIbl5ndNZY1MxM1CZmYdUWYimA+MkjRS0jZkX/az6itJehOwIzC3xFjMzKwHpSWCiFgHnADMAe4GLo+IRZJOl3RErupEYEZERFmxmJlZz0q7fBQgImYDs+vKTqubn1pmDGZm1lxv6Sw2M7MOcSIwM6s4JwIzs4pzIjAzq7iNJgJJr21HIGZm1hlFzghuknSFpMMkqfSIzMysrYokgj2A6cAngD9J+g9Je5QblpmZtctGE0FkromIY4DjgWOBeZKul3Rg6RGamVmpNnpDWeoj+EeyM4JHgBPJhorYB7iCbNRQMzPro4rcWTwXuASYEBHLc+ULJJ1bTlhmZtYuRRLB6J7GAYqIM1scj5mZtVmRzuJfSxpcm5G0o6Q55YVkZmbtVCQRdEfEqtpMRDwJvK60iMzMrK2KJIL1kobXZtJjJT1ktJlZP1Gkj+DrwB8kXQ8IeA8wqdSozMysbTaaCCLiV5L2BQ5IRV+MiMfKDcvMzNql6INp1gOPAgOBPSUREb8rLywzM2uXIjeUfRo4iezh87eTnRnMBQ4uNTIzM2uLIp3FJwHvAO6PiPcBY4BVZQZlZmbtUyQRPBcRzwFI2jYi/giMLjcsMzNrlyJ9BMvTDWVXAddIehK4v8ygzMysfYpcNfThNDlV0nXADsCvSo3KzMzapmkikNQFLIqINwFExPVticrMzNqmaR9BRKwHluTvLDYzs/6lSB/BjsAiSfOAZ2qFEXFEaVGZmVnbFEkEp27uyiUdCnwf6ALOj4jvNKhzNDCVbPyiOyLi45u7PTMz23RFOos3q18g9S9MAw4BlgPzJc2KiMW5OqOAKcC7IuJJSR7V1MyszTZ6H4Gk1ZKeTq/nJK2X9HSBde8HLI2IeyNiLTADGF9X53hgWhramoh4dFM/gJmZbZkiZwTb1aYliezL/ICe37HBUODB3PxyYP+6Onuk9d5A1nw0NSJecWmqpEmkEU+HD3e/tZlZKxW5s3iDyFwFjGvR9rcGRgEHAccA5+Wfhpbb7vSIGBsRY7u7u1u0aTMzg2KDzh2Zm90KGAs8V2DdK4BhufldU1necuDmiHgBuE/SPWSJYX6B9ZuZWQsUuWro8Nz0OmAZr2zrb2Q+MErSSLIEMBGovyLoKrIzgR9JGkLWVHRvgXWbmVmLFOkjOG5zVhwR6ySdAMwha/+/ICIWSTodWBARs9Kyv5O0mOyZB5Mj4vHN2Z6ZmW2eIk1DFwEn1R5gL2lH4OyI+NTG3hsRs4HZdWWn5aYDODm9zMysA4p0Fr+1lgQA0qWeY0qLyMzM2qpIItgqnQUAIGknij/i0szMerkiX+hnA3MlXZHmPwqcUV5IZmbWTkU6iy+WtICXnlF8ZH6YCDMz69uKdBYfQPZMgv9K89tL2j8ibi49OjMzK12RPoIfAn/Nzf81lZmZWT9QJBEoXeYJQES8iDuLzcz6jSKJ4F5JX5A0IL1Ownf/mpn1G0USwWeBd5INE1EbQfT4MoMyM7P2KXLV0KNk4wQBIGkQ8CHgih7fZGZmfUahYagldUk6TNIlwH3Ax8oNy8zM2qXpGYGk95KNGHoYMA94F/CGiHi2DbGZmVkb9JgIJC0HHiC7VPQrEbFa0n1OAmZm/UuzpqGZwC5kzUCHS3o1EE3qm5lZH9RjIoiILwIjycYaOghYAnRLOlrSa9oSnZmZla5pZ3F6RvF1ETGJLCkcQ/Z0smVtiM3MzNqg8B3C6bnCVwNXp0tIzcysHyh0+Wi9iFjT6kDMzKwzNisRmJlZ/+FEYGZWcUWeR7AHMBnYLV8/Ig7u8U1mZtZnFOksvgI4FzgPWF9uOGZm1m5FEsG6iPCDaMzM+qkifQQ/l/Q5STtL2qn2Kj0yMzNriyJnBMemfyfnygJ4Q+vDMTOzdivyPIKR7QjEzMw6Y6NNQ+nxlF+QNDO9TpA0oMjKJR0qaYmkpZJOabD8k5JWSro9vT69OR/CzMw2X5GmoR8CA4D/TvOfSGVNv7QldQHTgEPIHnE5X9KsiFhcV/WyiDhhk6I2M7OWKZII3hERb8vNXyvpjgLv2w9YGhH3AkiaQTZgXX0iMDOzDipy1dB6SW+szUh6A8XuJxgKPJibX57K6n1E0p2p2WlYoxVJmiRpgaQFK1euLLBpMzMrqkgimAxcJ+m3kq4HrgW+3KLt/xwYERFvBa4BLmpUKSKmR8TYiBjb3d3dok2bmRkUu2roN5JGAaNT0ZKIeL7AulcA+V/4u6ay/Lofz82eD3y3wHrNzKyFmj2z+OCIuFbSkXWLdpdERFy5kXXPB0ZJGkmWACYCH6/bxs4R8XCaPQK4e9PCNzOzLdXsjOC9ZM1AhzdYFkDTRBAR6ySdAMwBuoALImKRpNOBBRExC/iCpCOAdcATwCc3/SOYmdmW6DERRMQ30+TpEXFffln6lb9RETEbmF1XdlpuegowpXC0ZmbWckU6i/+nQdnMVgdiZmad0ayP4E3AXsAOdf0E2wMDyw7MzMzao1kfwWjgQ8BgXt5PsBo4vsSYzMysjZr1EfwM+JmkAyNibhtjMjOzNioyxMRtkj5P1ky0oUkoIj5VWlRmZtY2RTqLLwH+BhgHXE92Y9jqMoMyM7P2KZIIdo+IU4FnIuIi4O+B/csNy8zM2qVIIngh/btK0luAHYDXlReSmZm1U5E+gumSdgROBWYBrwFOa/4WMzPrK4oMOnd+mrweP6fYzKzfaXZD2cnN3hgR57Q+HDMza7dmZwTbpX9HA+8gaxaC7OayeWUGZWZm7dPshrJ/A5D0O2DfiFid5qcCv2hLdGZmVroiVw29Hlibm1+byszMrB8octXQxcA8ST9N8xOAC8sKyMzM2qvIVUNnSPol8J5UdFxE3FZuWGZm1i7NrhraPiKelrQTsCy9ast2iognyg/PzMzK1uyM4FKyYahvIXs0ZY3SvO8pMDPrB5pdNfSh9G+hx1KamVnf1KxpaN9mb4yIW1sfjpmZtVuzpqGzmywL4OAWx2JmZh3QrGnofe0MxMzMOqPIfQSk4af35OVPKLu4rKDMzKx9NpoIJH0TOIgsEcwGPgj8gexGMzMz6+OKDDFxFPB+4C8RcRzwNrKH05iZWT9QJBGsiYgXgXWStgceBYYVWbmkQyUtkbRU0ilN6n1EUkgaWyxsMzNrlSJ9BAskDQbOI7u57K/A3I29SVIXMA04BFgOzJc0KyIW19XbDjgJuHnTQjczs1bo8YxA0jRJ74qIz0XEqog4l+xL/djURLQx+wFLI+LeiFgLzADGN6j3LeBM4LnNiN/MzLZQs6ahe4D/lLRM0ncljYmIZRFxZ8F1DwUezM0vT2UbpJvWhkVE0+cbSJokaYGkBStXriy4eTMzK6LHRBAR34+IA4H3Ao8DF0j6o6RvStpjSzcsaSvgHODLG6sbEdMjYmxEjO3u7t7STZuZWc5GO4sj4v6IODMixgDHkD2P4O4C617ByzuVd01lNdsBbwF+K2kZcAAwyx3GZmbttdFEIGlrSYdL+jHwS2AJcGSBdc8HRkkaKWkbYCIvPfeYiHgqIoZExIiIGAHcBBwREQs254OYmdnmaTbo3CFkZwCHkT2sfgYwKSKeKbLiiFgn6QRgDtAFXBARiySdDiyIiFnN12BmZu3Q7PLRKWTPJPhyRDy5OSuPiNlkdyPny07roe5Bm7MNMzPbMs0GnfPoomZmFVDkzmIzM+vHnAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4pwIzMwqzonAzKziSk0Ekg6VtETSUkmnNFj+WUkLJd0u6Q+S9iwzHjMze6XSEoGkLmAa8EFgT+CYBl/0l0bE3hGxD/Bd4Jyy4jEzs8bKPCPYD1gaEfdGxFpgBjA+XyEins7NvhqIEuMxM7MGti5x3UOBB3Pzy4H96ytJ+jxwMrANcHCjFUmaBEwCGD58eMsDNTOrso53FkfEtIh4I/A14Bs91JkeEWMjYmx3d3d7AzQz6+fKTAQrgGG5+V1TWU9mABNKjMfMzBooMxHMB0ZJGilpG2AiMCtfQdKo3OzfA38qMR4zM2ugtD6CiFgn6QRgDtAFXBARiySdDiyIiFnACZI+ALwAPAkcW1Y8ZmbWWJmdxUTEbGB2XdlpuemTyty+mZltXMc7i83MrLOcCMzMKs6JwMys4pwIzMwqzonAzKzinAjMzCrOicDMrOKcCMzMKs6JwMys4kq9s7i3uOq2FZw1ZwkPrVrDLoMHMXncaCaMGdrpsMzMeoV+nwiuum0FU65cyJoX1gOwYtUaply5EMDJwCzxj6Vq6/eJ4Kw5SzYkgZo1L6znqzPv5CfzHmjZdhY//DR77rx9y9Zn1i7+sWT9vo/goVVrGpavXf9iS7ez587bM34f/9FY39PTj6Wz5izpUETWbv3+jGCXwYNY0SAZDB08iMs+c2AHIjLrXXr6sdRTufU//f6MYPK40Qwa0PWyskEDupg8bnSHIjLrXXYZPGiTyq3/6feJYMKYoXz7yL0ZOngQIjsT+PaRe7vt0yzxjyXr901DkCUDf/GbNVb72/BVQ9VViURgZs35x1K19fumITMza86JwMys4pwIzMwqzonAzKzinAjMzCpOEdHpGDaJpJXA/Z2OYwsNAR7rdBC9gPdDxvsh4/1Q7j7YLSK6Gy3oc4mgP5C0ICLGdjqOTvN+yHg/ZLwfOrcP3DRkZlZxTgRmZhXnRNAZ0zsdQC/h/ZDxfsh4P3RoH7iPwMys4nxGYGZWcU4EZmYV50RQAkkXSHpU0l25sqmSVki6Pb0Oyy2bImmppCWSxnUm6taSNEzSdZIWS1ok6aRUvpOkayT9Kf27YyqXpB+k/XCnpH07+wlao8l+qNrxMFDSPEl3pP3wb6l8pKSb0+e9TNI2qXzbNL80LR/R0Q/QIk32w4WS7ssdD/uk8vb8XUSEXy1+AX8L7AvclSubCnylQd09gTuAbYGRwJ+Brk5/hhbsg52BfdP0dsA96bN+FzgllZ8CnJmmDwN+CQg4ALi505+h5P1QteNBwGvS9ADg5vT/fDkwMZWfC/xLmv4ccG6anghc1unPUPJ+uBA4qkH9tvxd+IygBBHxO+CJgtXHAzMi4vmIuA9YCuxXWnBtEhEPR8StaXo1cDcwlOzzXpSqXQRMSNPjgYsjcxMwWNLO7Y269Zrsh5701+MhIuKvaXZAegVwMDAzldcfD7XjZCbwfklqT7TlabIfetKWvwsngvY6IZ3eXVBrEiH7UngwV2c5zb8o+px0Wj+G7NfP6yPi4bToL8Dr03TV9gNU7HiQ1CXpduBR4Bqys51VEbEuVcl/1g37IS1/CnhtWwMuSf1+iIja8XBGOh6+J2nbVNaW48GJoH1+CLwR2Ad4GDi7o9G0iaTXAP8DfDEins4vi+zctxLXLzfYD5U7HiJifUTsA+xKdpbzps5G1Bn1+0HSW4ApZPvjHcBOwNfaGZMTQZtExCPpAHgROI+XTvdXAMNyVXdNZX2epAFkX34/jogrU/EjtVPb9O+jqbxS+6GKx0NNRKwCrgMOJGvqqD0yN/9ZN+yHtHwH4PH2Rlqu3H44NDUhRkQ8D/yINh8PTgRtUteu92GgdkXRLGBiukpiJDAKmNfu+Fottef+P+DuiDgnt2gWcGyaPhb4Wa78n9JVEgcAT+WakPqsnvZDBY+HbkmD0/Qg4BCy/pLrgKNStfrjoXacHAVcm84g+7Qe9sMfcz+ORNZPkj8eSv+78MPrSyDpJ8BBwBBJy4FvAgelS8ICWAZ8BiAiFkm6HFgMrAM+HxHrOxB2q70L+ASwMLWHAvwr8B3gckn/TDac+NFp2WyyKySWAs8Cx7U12vL0tB+OqdjxsDNwkaQush+gl0fE1ZIWAzMk/TtwG1nSJP17iaSlZBdeTOxE0CXoaT9cK6mb7Oqg24HPpvpt+bvwEBNmZhXnpiEzs4pzIjAzqzgnAjOzinMiMDOrOCcCM7OKcyKwXiXdXv/F3PwcSefn5s+WdHKT918o6ag0/VtJr3gQuKQBkr6jbATUWyXNlfTBtGyZpCGbEfeG7fawfFoaVXKxpDW5USaPkjS7dm15K0naWdLVTZZvI+l3uRu6rKKcCKy3uQF4J4CkrYAhwF655e8EbtzCbXyL7Hrut0TEvmQ38Gy3hetsKiI+n4YVOAz4c0Tsk14zI+KwdJdpq51MdtdyTzGtBX4DfKyEbVsf4kRgvc2NZEMPQJYA7gJWS9oxDcT1ZuBWSadJmi/pLknTi45MKelVwPHAiel2/tpwD5c3qHtyWv9ddWcp/5QGB7tD0iUN3vetdIbQVTCmZZKGSBoh6Y/pvfdI+rGkD0i6IZ297Jfqv1rZQHXzJN0maXwPq/4I8Kv0nr1S/dtT7KNSnauAfygSp/VfPiW0XiUiHpK0TtJwsl//c8lGWzyQbATKhRGxVtJ/RcTpAOnL+EPAzwtsYnfggfoB8OpJejvZXZz7k93tebOk64G1wDeAd0bEY5J2qnvfWWRnF8dt5pAIuwMfBT4FzAc+DrwbOILsjuQJwNfJhlz4VGpSmifpfyPimVwcI4Ena8mO7E7V70fEj5U9/KWWpO4iG+jMKsxnBNYb3UiWBGqJYG5u/oZU533Knly1kGxM+70arWgLvBv4aUQ8k8aPvxJ4T9rWFRHxGEBE5J87cSqwQ0R8dgvGxbkvIhamwegWAb9J61oIjEh1/g44JQ1Z8VtgIDC8bj07Aytz83OBf5X0NWC3iFiT4l8PrJVUatOY9W5OBNYb1foJ9ib7xXoT2RnBO4EbJQ0E/pvsiU57k7WDDyy47qXAcEnbtzzq7Bf82+vPEjbR87npF3PzL/LSGbyAj+T6GYZHxN1161lDbp9ExKVkZxVrgNmSDs7V3RZ4bgtitj7OicB6oxvJmnqeSEM1PwEMJksGN/LSF9xjysb57/FqnXoR8SzZgGbf10vPx+2W9NG6qr8HJkh6laRXk40Q+nvgWuCjkl6b3pv/0v8V2aB6vyj5F/Yc4MRav4ikMQ3q3MNLZxBIegNwb0T8gGyEz7em8tcCj0XECyXGa72cE4H1RgvJrha6qa7sqYh4LF1hcx7Z2cIcsl/im+IbZM0miyXdBVwN1D8051ay58jOI3ui2PkRcVtELALOAK6XdAdwTt37rkixzVI2zHAZvkX2iMM7JS1K8y+T+gv+LGn3VHQ0cFdqTnoLcHEqfx/wi5LitD7Co4+a9VOSPgy8PSK+0aTOlcApEXFP+yKz3sZXDZn1UxHx01oTViOpaewqJwHzGYGZWcW5j8DMrOKcCMzMKs6JwMys4pwIzMwqzonAzKzi/g/E4r9dHt+9nQAAAABJRU5ErkJggg==",+ "text/plain": [+ "<Figure size 432x288 with 1 Axes>"+ ]+ },+ "metadata": {+ "needs_background": "light"+ },+ "output_type": "display_data"+ }+ ],+ "source": [+ "from flaml.data import get_output_from_log\n",+ "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n",+ " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n",+ "for config in config_history:\n",+ " print(config)\n",+ "\n",+ "import matplotlib.pyplot as plt\n",+ "import numpy as np\n",+ "plt.title('Learning Curve')\n",+ "plt.xlabel('Wall Clock Time (s)')\n",+ "plt.ylabel('Validation Accuracy')\n",+ "print(len(valid_loss_history))\n",+ "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n",+ "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### 3.2 Text Summarization Example"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "The text summarization task summarizes a long text into a short sentence. For example:\n",+ "\n",+ "- Document: Army explosives experts were called out to deal with a suspect package at the offices on the Newtownards Road on Friday night. Roads were sealed off and traffic diverted as a controlled explosion was carried out. The premises, used by East Belfast MP Naomi Long, have been targeted a number of times. Most recently, petrol bomb attacks were carried out on the offices on consecutive nights in April and May. The attacks began following a Belfast City Council vote in December 2012 restricting the flying of the union flag at the City Hall. Condemning the latest hoax, Alliance MLA Chris Lyttle said: \"It is a serious incident for the local area, it causes serious disruption, it puts people's lives at risk, it can prevent emergency services reaching the area. \"Ultimately we need people with information to share that with the police in order for them to do their job and bring these people to justice.\n",+ "\n",+ "- Summary: A suspicious package left outside an Alliance Party office in east Belfast has been declared a hoax.\n",+ "\n",+ "In this example, we use FLAML to perform *abstractive summarization* using the t5-small language model, i.e., the summary is generated word-by-word. "+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 15,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Using custom data configuration default\n",+ "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "204045\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Using custom data configuration default\n",+ "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n",+ "Using custom data configuration default\n",+ "Reusing dataset xsum (/home/xliu127/.cache/huggingface/datasets/xsum/default/1.2.0/32c23220eadddb1149b16ed2e9430a05293768cfffbdfd151058697d4c11f934)\n"+ ]+ }+ ],+ "source": [+ "from datasets import load_dataset\n",+ "\n",+ "train_dataset = load_dataset(\"xsum\", split=\"train\").to_pandas()\n",+ "print(len(train_dataset))\n",+ "dev_dataset = load_dataset(\"xsum\", split=\"validation\").to_pandas()\n",+ "test_dataset = load_dataset(\"xsum\", split=\"test\").to_pandas()\n",+ "\n",+ "custom_sent_keys = [\"document\"] # specify the column names of the input sentences\n",+ "label_key = \"summary\" # specify the column name of the label \n",+ "\n",+ "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n",+ "X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]\n",+ "X_test = test_dataset[custom_sent_keys]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 16,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/html": [+ "== Status ==<br>Current time: 2022-03-19 14:55:00 (running for 00:08:31.38)<br>Memory usage on this node: 23.1/376.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/250.17 GiB heap, 0.0/111.21 GiB objects (0.0/1.0 accelerator_type:V100)<br>Current best trial: 08b6571c with val_loss=0.8569452656271894 and parameters={'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 9223372036854775807, 'learner': 'transformer', 'FLAML_sample_size': 10000}<br>Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-03-19_14-46-29<br>Number of trials: 8/1000000 (8 TERMINATED)<br><br>"+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m warnings.warn(\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'loss': 8.7635, 'learning_rate': 1.2308416834153697e-05, 'epoch': 0.11}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m warnings.warn(\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m warnings.warn(\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m warnings.warn(\n",+ "2022-03-19 14:56:00,679\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 102.2734, 'eval_samples_per_second': 110.801, 'eval_steps_per_second': 6.932, 'epoch': 0.11}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:57:00,687\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 107.4032, 'eval_samples_per_second': 105.509, 'eval_steps_per_second': 6.601, 'epoch': 0.16}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.2143, 'eval_samples_per_second': 104.718, 'eval_steps_per_second': 6.552, 'epoch': 0.36}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:58:00,697\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 109.4667, 'eval_samples_per_second': 103.52, 'eval_steps_per_second': 6.477, 'epoch': 0.38}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 110.7246, 'eval_samples_per_second': 102.344, 'eval_steps_per_second': 6.403, 'epoch': 0.11}\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'train_runtime': 220.8946, 'train_samples_per_second': 4.648, 'train_steps_per_second': 0.149, 'train_loss': 8.763471198804451, 'epoch': 0.11}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 14:59:00,706\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m Num examples = 11332\n",+ "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m Batch size = 16\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 109.1975, 'eval_samples_per_second': 103.775, 'eval_steps_per_second': 6.493, 'epoch': 0.16}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m {'train_runtime': 232.9303, 'train_samples_per_second': 10.067, 'train_steps_per_second': 1.262, 'train_loss': 9.880440506280637, 'epoch': 0.16}\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.3182, 'eval_samples_per_second': 104.618, 'eval_steps_per_second': 6.546, 'epoch': 0.36}\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m {'train_runtime': 232.4568, 'train_samples_per_second': 92.218, 'train_steps_per_second': 2.887, 'train_loss': 11.215172903878349, 'epoch': 0.36}\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 110.526, 'eval_samples_per_second': 102.528, 'eval_steps_per_second': 6.415, 'epoch': 0.38}\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m {'train_runtime': 236.6253, 'train_samples_per_second': 19.714, 'train_steps_per_second': 0.621, 'train_loss': 11.549961930614407, 'epoch': 0.38}\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2022-03-19 15:00:00,942\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m Num examples = 11332\n",+ "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m Batch size = 16\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m Num examples = 11332\n",+ "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m Batch size = 16\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m Using amp half precision backend\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m ***** Running Prediction *****\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m Num examples = 11332\n",+ "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m Batch size = 16\n",+ "2022-03-19 15:01:00,948\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n",+ "2022-03-19 15:02:20,150\tINFO tune.py:639 -- Total run time: 950.87 seconds (500.36 seconds for the tuning loop).\n",+ "[flaml.automl: 03-19 15:02:25] {2837} INFO - selected model: None\n",+ "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",+ " warnings.warn(\n",+ "[flaml.automl: 03-19 15:14:54] {2947} INFO - retrain transformer for 748.2s\n",+ "[flaml.automl: 03-19 15:14:54] {2954} INFO - retrained model: None\n",+ "[flaml.automl: 03-19 15:14:54] {2283} INFO - fit succeeded\n",+ "[flaml.automl: 03-19 15:14:54] {2284} INFO - Time taken to find the best model: 472.3055913448334\n",+ "[flaml.automl: 03-19 15:14:54] {2295} WARNING - Time taken to find the best model is 94% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'train_runtime': 14.6848, 'train_samples_per_second': 13894.959, 'train_steps_per_second': 434.258, 'train_loss': 10.199760437011719, 'epoch': 0.02}\n"+ ]+ }+ ],+ "source": [+ "''' import AutoML class from flaml package '''\n",+ "from flaml import AutoML\n",+ "automl = AutoML()\n",+ "\n",+ "import ray\n",+ "if not ray.is_initialized():\n",+ " ray.init()\n",+ "\n",+ "automl_settings = {\n",+ " \"time_budget\": 500, # setting the time budget\n",+ " \"task\": \"summarization\", # setting the task as multiplechoice-classification\n",+ " \"hf_args\": {\n",+ " \"output_dir\": \"data/output/\", # setting the output directory\n",+ " \"ckpt_per_epoch\": 1, # setting the number of checkoints per epoch\n",+ " \"model_path\": \"t5-small\",\n",+ " \"per_device_eval_batch_size\": 16, # the batch size for validation (inference)\n",+ " },\n",+ " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n",+ " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n",+ " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n",+ " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n",+ " \"metric\": \"rouge1\",\n",+ " \"n_concurrent_trials\": 4, # sample: False # if the time is sufficient (e.g., longer than one trial's running time), you can set \n",+ "}\n",+ "\n",+ "'''The main flaml automl API'''\n",+ "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 17,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 3.6439277745413994e-06, 'num_train_epochs': 0.454119690781029, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.04654549348562217, 'weight_decay': 0.06669806327326033, 'adam_epsilon': 2.5833461668835812e-08, 'seed': 42, 'global_max_steps': 125, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 3.6439277745413994e-06, 'num_train_epochs': 0.454119690781029, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.04654549348562217, 'weight_decay': 0.06669806327326033, 'adam_epsilon': 2.5833461668835812e-08, 'seed': 42, 'global_max_steps': 125, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 3.4236378229097798e-06, 'num_train_epochs': 8.919336644807531, 'per_device_train_batch_size': 4, 'warmup_ratio': 0.022492820063166875, 'weight_decay': 0.27013721375576616, 'adam_epsilon': 6.366959214432801e-08, 'seed': 43, 'global_max_steps': 180, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n",+ "{'Current Learner': 'transformer', 'Current Sample': 10000, 'Current Hyper-parameters': {'learning_rate': 2.83823390666728e-06, 'num_train_epochs': 1.6667827812145841, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.04013366246992448, 'weight_decay': 0.2945152447208819, 'adam_epsilon': 4.694476379503266e-08, 'seed': 43, 'global_max_steps': 163, 'learner': 'transformer', 'FLAML_sample_size': 10000}, 'Best Learner': 'transformer', 'Best Hyper-parameters': {'learning_rate': 1.0000000000000003e-05, 'num_train_epochs': 1.0, 'per_device_train_batch_size': 32, 'warmup_ratio': 0.0, 'weight_decay': 0.0, 'adam_epsilon': 1e-06, 'seed': 42, 'global_max_steps': 112, 'learner': 'transformer', 'FLAML_sample_size': 10000}}\n",+ "4\n"+ ]+ },+ {+ "data": {+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEWCAYAAACjYXoKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAcdUlEQVR4nO3de7gddX3v8feHcAtFiEqkEEBQKApVQQIUKx5vPUBVwAoKeNpGq6gU2+ojFmq1FsoRS2nVSlX0WGorClKM2EajvSlF0UTAQKDRSFNIQm1QuWkkBL7nj5mNi83eO2tn9tqX5P16nvUw85vfzHz3hL0++zez1kyqCkmSuthmqguQJM18hokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0ykCZTk6CQrproOabIZJtpiJFmV5MVTWUNVXVNVBw5q+0mOSfLVJPclWZfkK0mOH9T+pH4ZJtI4JJk1hfs+CfgM8AlgL2B34F3AyzZjW0ni778mjP8zaYuXZJskZyf5XpIfJLkiyRN6ln8myX8nuaf9q//gnmWXJvlQkkVJfgy8oB0BvS3Jsnady5Ps2PZ/fpLVPeuP2rdd/vYkdyZZm+R1SSrJ/iP8DAH+HDivqj5WVfdU1cNV9ZWqen3b591J/q5nnX3b7W3bzv9bkvOTXAv8BDgrydJh+3lLkqvb6R2S/FmS25N8P8mHk8zu+M+hLZRhoq3Bm4ETgf8F7An8CLi4Z/kXgAOAJwHXA58ctv5pwPnA44B/b9teCRwL7Ac8E1gwxv5H7JvkWOCtwIuB/YHnj7GNA4G9gSvH6NOPXwdOp/lZPgwcmOSAnuWnAZe10xcAvwAc0tY3j2YkJD2GYaKtwRuBd1TV6qp6AHg3cNLQX+xV9fGquq9n2bOS7Nqz/ueq6tp2JPDTtu0DVbW2qn4IfJ7mDXc0o/V9JfDXVbW8qn7S7ns0T2z/e2d/P/KoLm33t7Gq7gE+B5wK0IbK04Cr25HQ6cBbquqHVXUf8H+BUzruX1sow0RbgycDn01yd5K7gVuBh4Ddk8xKckF7CuxeYFW7zm49698xwjb/u2f6J8DOY+x/tL57Dtv2SPsZ8oP2v3uM0acfw/dxGW2Y0IxKFrbBNhfYCfhWz3H7YtsuPYZhoq3BHcBxVTWn57VjVa2heQM9geZU067Avu066Vl/ULfWvpPmQvqQvcfou4Lm53jFGH1+TBMAQ35+hD7Df5YvA3OTHEITKkOnuO4C1gMH9xyzXatqrNDUVsww0ZZmuyQ79ry2pbk2cH6SJwMkmZvkhLb/44AHaP7y34nmVM5kuQJ4TZKnJ9kJeOdoHat5VsRbgXcmeU2SXdoPFjw3ySVttxuB5yXZpz1Nd86mCqiqB2k+IXYh8ASacKGqHgY+CvxFkicBJJmX5JjN/WG1ZTNMtKVZRPMX9dDr3cD7gauBLyW5D7gOOLLt/wngv4A1wC3tsklRVV8APgD8K7CyZ98PjNL/SuBVwGuBtcD3gT+hue5BVX0ZuBxYBnwL+Ic+S7mMZmT2mara2NP++0N1tacA/4nmgwDSY8SHY0nTQ5KnAzcDOwx7U5emPUcm0hRK8vL2+xyPB94LfN4g0UxkmEhT6w3A/wDfo/mE2Zumthxp83iaS5LUmSMTSVJn2051AZNht912q3333Xeqy5CkGWO33XZj8eLFi6vq2H76bxVhsu+++7J06dJNd5QkPSLJbpvu1fA0lySpM8NEktSZYSJJ6swwkSR1ZphIkjrbKj7NJUnjsfCGNVy4eAVr717PnnNmc9YxB3LiofOmuqxpzTCRpB4Lb1jDOVfdxPoHHwJgzd3rOeeqmwAMlDEYJpLU48LFKx4JkiHrH3yIt1+5jE998/YpqmrzHLTnLvzRyw6elH0N9JpJkmOTrEiyMsnZIyx/XpLrk2xMctIIy3dJsjrJB3vavpjk20mWJ/lwklmD/BkkbV3W3r1+xPYNDz08yZXMLAMbmbRv8hcDvwKsBpYkubqqbunpdjuwAHjbKJs5D/jqsLZXVtW9SQJcCZwMfHoia5e09dpzzmzWjBAo8+bM5vI3HDUFFc0MgxyZHAGsrKrbqmoDzRv+Cb0dqmpVVS0DHhP5SQ4Ddge+NGyde9vJbYHtGdzzuSVthc465kBmb/foEx6zt5vFWcf4kMmxDDJM5gF39Myvbts2Kck2wEWMMmJJspjmGRD30YxORupzepKlSZauW7duPHVL2oqdeOg83vNrz2D7Wc3b47w5s3nPrz3Di++bMF2/Z3IGsKiqVo+0sKqOAfYAdgBeOEqfS6pqflXNnzt37uAqlbTFOfHQeRy6zxyO3O8JXHv2Cw2SPgzy01xrgL175vdq2/pxFHB0kjOAnYHtk9xfVY9cxK+qnyb5HM2psy9PUM2SpM0wyDBZAhyQZD+aEDkFOK2fFavq1UPTSRYA86vq7CQ7A4+rqjuTbAu8BLhmwiuXJI3LwE5zVdVG4ExgMXArcEVVLU9ybpLjAZIcnmQ1zSeyPpJk+SY2+3PA1UmWATfSXDf58KB+BklSfwb6pcWqWgQsGtb2rp7pJTSnv8baxqXApe3094HDJ7pOSVI30/UCvCRpBjFMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4GGiZJjk2yIsnKJGePsPx5Sa5PsjHJSSMs3yXJ6iQfbOd3SvKPSf4jyfIkFwyyfklSfwYWJklmARcDxwEHAacmOWhYt9uBBcBlo2zmPOCrw9r+rKqeBhwK/HKS4yasaEnSZhnkyOQIYGVV3VZVG4BPAyf0dqiqVVW1DHh4+MpJDgN2B77U0/8nVfWv7fQG4Hpgr8H9CJKkfgwyTOYBd/TMr27bNinJNsBFwNvG6DMHeBnwz6MsPz3J0iRL161b12/NkqTNMF0vwJ8BLKqq1SMtTLIt8CngA1V120h9quqSqppfVfPnzp07wFIlSdsOcNtrgL175vdq2/pxFHB0kjOAnYHtk9xfVUMX8S8BvltV75uoYiVJm2+QYbIEOCDJfjQhcgpwWj8rVtWrh6aTLADmDwVJkj8BdgVeN9EFS5I2z8BOc1XVRuBMYDFwK3BFVS1Pcm6S4wGSHJ5kNXAy8JEky8faZpK9gHfQfDrs+iQ3JjFUJGmKDXJkQlUtAhYNa3tXz/QSNvFprKq6FLi0nV4NZKLrlCR1M10vwEuSZhDDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdbZZYZLkCxNdiCRp5tp2tAVJnj3aIuCQgVQjSZqRRg0TYAnwFZrwGG7OQKqRJM1IY4XJrcAbquq7wxckuWNwJUmSZpqxrpm8e4zlb574UiRJM9WoI5OqunKMZQsHUo0kaUbyo8GSpM4ME0lSZwMNkyTHJlmRZGWSs0dY/rwk1yfZmOSkEZbvkmR1kg/2tJ2f5I4k9w+ydklS/zYZJkl2SvLOJB9t5w9I8tI+1psFXAwcBxwEnJrkoGHdbgcWAJeNspnzgK8Oa/s8cMSm9i9Jmjz9jEz+GngAOKqdXwP8SR/rHQGsrKrbqmoD8GnghN4OVbWqqpYBDw9fOclhwO7Al4atc11V3dnH/iVJk6SfMHlqVf0p8CBAVf2Ekb/IONw8oPf7KKvbtk1Ksg1wEfC2fvqPso3TkyxNsnTdunWbuxlJUh/6CZMNSWYDBZDkqTQjlUE6A1hUVas3dwNVdUlVza+q+XPnzp3A0iRJw431DfghfwR8Edg7ySeBX6a5zrEpa4C9e+b3atv6cRRwdJIzgJ2B7ZPcX1WPuYgvSZp6mwyTqvpykuuBX6I5vfW7VXVXH9teAhyQZD+aEDkFOK2foqrq1UPTSRYA8w0SSZq++vk017OBJwN3AmuBfZI8NcmYQVRVG4EzgcU09/m6oqqWJzk3yfHttg9Psho4GfhIkuV91POn7To7tR8bfvem1pEkDVY/p7n+Cng2sIxmZPKLwHJg1yRvqqovjbZiVS0CFg1re1fP9BKa01+jqqpLgUt75t8OvL2PuiVJk6SfC/BrgUPbi9mHAYcCtwG/AvzpIIuTJM0M/YTJL1TVI6efquoW4GlVddvgypIkzST9nOZanuRDNF86BHgVcEuSHWi/eyJJ2rr1MzJZAKwEfq993da2PQi8YDBlSZJmkn4+Grye5tvoF42w2JstSpI2HSZJ/pP22++9quopA6lIkjTj9HPNZH7P9I403wl5wmDKkSTNRJu8ZlJVP+h5ramq9wEvGXxpkqSZop/TXM/umd2GZqTSz4hGkrSV6CcUei+8bwRWAa8cSDWSpBmpn09z+fFfSdKY+rnR465J/nzoQVNJLkqy62QUJ0maGfr50uLHgftoTm29EriX5lG+kiQB/V0zeWpVvaJn/o+T3DigeiRJM1A/I5P1SZ47NJPkl4H1gytJkjTT9DMyeRPwN+11kgA/BH5zoFVJkmaUfj7NdSPwrCS7tE0/pnkE77IB1iVJmkFGPc2VZJck5yT5YJJfobkI/xs0dxD2eyaSpEeMNTL5W+BHwNeB1wPvoDnN9fJ2tCJJEjB2mDylqp4BkORjwJ3APlX100mpTJK0WRbesIYLF69g7d3r2XPObM465kBOPHTeQPc5Vpg88hTFqnooyWqDRJKmt4U3rOGcq25i/YMPAbDm7vWcc9VNAAMNlLE+GvysJPe2r/uAZw5NJ7l3YBVJkjbbhYtXPBIkQ9Y/+BAXLl4x0P2OOjKpqlkD3bMkacKtvXvkrwGO1j5R+vnSoiRphthzzuxxtU8Uw0SStiBnHXMgs7d79Iml2dvN4qxjDhzofn3IlSRtQYYusk+nT3NJkmagEw+dN/DwGM7TXJKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgYaJkmOTbIiycokZ4+w/HlJrk+yMclJIyzfJcnqJB/saTssyU3tNj+QJIP8GSRJmzawMEkyC7gYOA44CDg1yUHDut0OLAAuG2Uz5wFfHdb2IZrHCB/Qvo6doJIlSZtpkCOTI4CVVXVbVW0APg2c0NuhqlZV1TLg4eErJzkM2B34Uk/bHsAuVXVdVRXwCeDEwf0IkqR+DPJGj/OAO3rmVwNH9rNikm2Ai4D/A7x42DZXD9vmiHczS3I6cDrAPvvs03fRQ6biGcqSNFNN1wvwZwCLqmr1JnuOoqouqar5VTV/7ty541p36BnKa+5eT/GzZygvvGHN5pYjSVu0QY5M1gB798zv1bb14yjg6CRnADsD2ye5H3h/u53N2WbfRnuG8tuvXManvnn7RO9O0jR0y533ctAeu0x1GTPGIMNkCXBAkv1o3vBPAU7rZ8WqevXQdJIFwPyqOrudvzfJLwHfAH4D+MsJrnvUZyVveOgxl3YkbaEO2mMXTjjEU9v9GliYVNXGJGcCi4FZwMeranmSc4GlVXV1ksOBzwKPB16W5I+r6uBNbPoM4FJgNvCF9jWh9pwzmzUjBMq8ObO5/A1HTfTuJGnGS/OhqC3b/Pnza+nSpX33H7pm0nuqa/Z2s3jPrz3Di/CSthpJvlVV8/vp62N7RzBVz1CWpJnKMBnFVDxDWZJmqun60WBJ0gximEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHU20DBJcmySFUlWJjl7hOXPS3J9ko1JTuppf3LbfmOS5Une2LPsVUmWte3vHWT9kqT+DCxMkswCLgaOAw4CTk1y0LButwMLgMuGtd8JHFVVhwBHAmcn2TPJE4ELgRdV1cHAzyd50aB+BklSfwY5MjkCWFlVt1XVBuDTwAm9HapqVVUtAx4e1r6hqh5oZ3foqfMpwHeral07/0/AKwb1A0iS+jPIMJkH3NEzv7pt60uSvZMsa7fx3qpaC6wEDkyyb5JtgROBvSeuZEnS5pi2F+Cr6o6qeiawP/CbSXavqh8BbwIuB64BVgEPjbR+ktOTLE2ydN26dSN1kSRNkEGGyRoePWrYq20bl3ZEcjNwdDv/+ao6sqqOAlYA3xllvUuqan5VzZ87d+64i5ck9W+QYbIEOCDJfkm2B04Bru5nxSR7JZndTj8eeC5NcJDkST3tZwAfG0DtkqRxGFiYVNVG4ExgMXArcEVVLU9ybpLjAZIcnmQ1cDLwkSTL29WfDnwjybeBrwB/VlU3tcven+QW4FrggqoacWQiSZo8qaqprmHg5s+fX0uXLp3qMiRpRknyraqa30/faXsBXpI0cxgmkqTODBNJUmeGiSSpM8NEktSZYSJJ6swwkSR1ZphIkjrbdqoLkCRNnIU3rOHCxStYe/d69pwzm7OOOZATD+37hu2bzTCRpC3EwhvWcM5VN7H+weZm6mvuXs85VzV3ohp0oHiaS5K2EBcuXvFIkAxZ/+BDXLh4xcD3bZhI0hZi7d3rx9U+kQwTSdpC7Dln9rjaJ5JhIklbiLOOOZDZ2816VNvs7WZx1jEHDnzfXoCXpC3E0EV2P80lSerkxEPnTUp4DOdpLklSZ4aJJKkzw0SS1JlhIknqzDCRJHWWqprqGgYuyTrgvwa4i92Auwa4/S6ma23WNT7WNT7WNX7Da7sLoKqO7WflrSJMBi3J0qqaP9V1jGS61mZd42Nd42Nd49e1Nk9zSZI6M0wkSZ0ZJhPjkqkuYAzTtTbrGh/rGh/rGr9OtXnNRJLUmSMTSVJnhokkqTPDpE9JZiW5Ick/DGv/QJL7e+Z3SHJ5kpVJvpFk32lU24Ik65Lc2L5eN5l1Jbk0yX/27P+Qtj1trSuTLEvy7GlS1/OT3NPT/q5JritJzk/ynSS3JvmdnvapPF6j1TXVx+uann2vTbKwp96pPF6j1TWpx2uU2l6U5Pp2//+eZP+2fdzvY96Cvn+/C9wK7DLUkGQ+8Phh/X4L+FFV7Z/kFOC9wKumSW0Al1fVmQOuZ9S6gLOq6sph/Y4DDmhfRwIfav871XUBXFNVLx1gLb2G17UA2Bt4WlU9nORJbftUH6/R6oIpPF5VdfTQgiR/D3yunZ3S4zVGXTC5x+sxtdEcixOq6tYkZwB/SPPvO+73MUcmfUiyF/AS4GM9bbOAC4G3D+t+AvA37fSVwIuSZJrUNmlGqmsMJwCfqMZ1wJwke0yDuibNKHW9CTi3qh4GqKr/adun+niNVtekGevfMckuwAuBhW3TVB+v0eqaVKPUVvwsWHYF1rbT434fM0z68z6aN+aHe9rOBK6uqjuH9Z0H3AFQVRuBe4AnTpPaAF7RDvWvTLL3JNcFcH67/79IskPb9sgxa61u26a6LoCjknw7yReSHDygmkar66nAq5Isbfd/QNs+1cdrtLpgao/XkBOBf66qe9v5qT5eo9UFk3e8RqvtdcCiJKuBXwcuaNvH/T5mmGxCkpcC/1NV3+pp2xM4GfjLKSuMzart88C+VfVM4Mv87C+PgdfVOgd4GnA48ATg9wex/wms63rgyVX1LJrjuXCS69oB+Gl7i4uPAh8fxP4nsK6pPl5DTgU+NYh9j2Uz6pqU47WJ2t4C/GpV7QX8NfDnm72TqvI1xgt4D81fMquA/wZ+AvyonV7Vvh4GVrb9FwNHtdPb0twsLdOhtmHrzgLumcS6/m5Yn+cD/9BOfwQ4tWfZCmCPqa5rhPVXAbtNVl3AfwD7tX0y9O811cdrtLqm+ni1y3YDfgDs2NN/yv//GqmuyTpeY9T2j8D3evrsA9zSTo/7fWzCi96SX6O90QD390z/NvDhdvoU4IppVNsePdMvB66bzLqG9t++Ab0PuKCdfwnwhbb9l4BvTpO6fn7oFwg4Arh9U79QE1zXBcBre9qXTJPjNVpdU3q82vk3An8zrM+UHq8x6pr049VbW09I/ELb/lvA37fT434f89NcE+//AX+bZCXwQ5p/iOnid5IcD2ykqW3BJO//k0nm0vxS30jzCwawCPhVYCXNX0yvmSZ1nQS8KclGYD1wSrW/XZPkgra2twD305zfhqk/XqPVNdXHC5rftwuGtU318YKR65rS41VVG5O8Hvj7JA/TnNV4bbt43O9j3k5FktSZF+AlSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWGiGa+9Bcrv9cwvTtJ7r7KLkrx1jPUvTXJSO/1v7U0yh/fZLskFSb7b3mX160mOa5etSrLbZtT9yH5HWX5xezfXW5Ks77m77ElJFiWZM9599lHTHhl29+lhy7dP8tUkfq1Aj2KYaEtwLfAcgCTb0HzbuPc+R88BvtZxH+cBewC/WFXPprnP0uM6bnNMVfXbVXUIzXckvldVh7SvK6vqV6vq7gHs9q00t0gZraYNwD8z+Dtha4YxTLQl+BpwVDt9MHAzcF+Sx7c3bXw6cH2SdyVZkuTmJJf0ezfnJDsBrwfeXFUPAFTV96vqihH6vrXd/s3DRku/0d5I8ttJ/naE9c5rRyqz+qxpVZLdkuyb5D/adb+T5JNJXpzk2nYUdUTb/+eSfDzJN9M8z+KEUTb9CuCL7ToHt/1vbGsfuqHjQuDV/dSprYdDVc14VbU2ycYk+9CMQr5Oc9fTo2judnpTVW1I8sGqOhegfUN/Kc3NLzdlf+D2evTdXh8jyWE0364+kubb9N9I8hVgA81zIp5TVXclecKw9S6kGeW8ZjO/Ab0/zc09XwssAU4DngscD/wBzSjqHcC/VNVr29Nj30zyT1X145469qN5hsUDbdMbgfdX1SeTbE9zPzdowvrwzahTWzBHJtpSfI0mSIbC5Os989e2fV6Q5qlxN9E8V2Kib/n9XOCzVfXjqrofuAo4ut3XZ6rqLoCq+mHPOu8Edq2qN3a4lcZ/VtVN1TxfZDnNbc4LuAnYt+3zv4Gzk9wI/BuwI82N/XrtAazrmf868AdJfp/m7rbr2/ofAjYkGehpPs0shom2FEPXTZ5B85fzdTQjk+cAX0uyI/BXwElV9Qya6wI79rntlcA+aR5uNNGWAIcNH62M0wM90w/3zD/Mz84+BHhFz3WXfarq1mHbWU/PMamqy2hGN+tpnnnxwp6+OwA/7VCztjCGibYUX6M5bfXDqnqo/et/Dk2gfI2fvUnelWRnmpvs9aWqfkJz47v3t6d7SDI3ycnDul4DnJhkpyQ/R3Nn5muAfwFOTvLEdt3e4PgizQ0A/3HAf+kvBt48dJ0oyaEj9PkOPxvJkOQpwG1V9QGaR80+s21/InBXVT04wHo1wxgm2lLcRPMpruuGtd1TVXe1n3z6KM2oZTHNiGA8/pDmFNAtSW6muYX3o66hVNX1wKXAN4FvAB+rqhuqajlwPvCVJN9m2AOIquozbW1XJ5k9zrr6dR6wHbAsyfJ2/lHa6yffS7J/2/RK4Ob21NgvAp9o219A8ywM6RHeNVjSI5K8HDisqv5wjD5XAWdX1XcmrzJNd36aS9IjquqzQ6fjRtKe5ltokGg4RyaSpM68ZiJJ6swwkSR1ZphIkjozTCRJnRkmkqTO/j/XoXUXWf2mTgAAAABJRU5ErkJggg==",+ "text/plain": [+ "<Figure size 432x288 with 1 Axes>"+ ]+ },+ "metadata": {+ "needs_background": "light"+ },+ "output_type": "display_data"+ }+ ],+ "source": [+ "\n",+ "from flaml.data import get_output_from_log\n",+ "time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n",+ " get_output_from_log(filename=automl_settings['log_file_name'], time_budget=3000)\n",+ "for config in config_history:\n",+ " print(config)\n",+ "\n",+ "import matplotlib.pyplot as plt\n",+ "import numpy as np\n",+ "plt.title('Learning Curve')\n",+ "plt.xlabel('Wall Clock Time (s)')\n",+ "plt.ylabel('Rouge 1')\n",+ "print(len(valid_loss_history))\n",+ "plt.scatter(time_history, 1 - np.array(valid_loss_history))\n",+ "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 18,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Using amp half precision backend\n",+ "***** Running Prediction *****\n",+ " Num examples = 1\n",+ " Batch size = 64\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Predicted labels ['prison link Cymru said some ex-offenders were living rough for up to ']\n"+ ]+ }+ ],+ "source": [+ "'''compute predictions of testing dataset''' \n",+ "y_pred = automl.predict([\"Prison Link Cymru had 1,099 referrals in 2015-16 and said some ex-offenders were living rough for up to a year before finding suitable accommodation.\\nWorkers at the charity claim investment in housing would be cheaper than jailing homeless repeat offenders.\\nThe Welsh Government said more people than ever were getting help to address housing problems.\\nChanges to the Housing Act in Wales, introduced in 2015, removed the right for prison leavers to be given priority for accommodation.\\nPrison Link Cymru, which helps people find accommodation after their release, said things were generally good for women because issues such as children or domestic violence were now considered.\\nHowever, the same could not be said for men, the charity said, because issues which often affect them, such as post traumatic stress disorder or drug dependency, were often viewed as less of a priority.\\nAndrew Stevens, who works in Welsh prisons trying to secure housing for prison leavers, said the need for accommodation was 'chronic'.\\n'There\\'s a desperate need for it, finding suitable accommodation for those leaving prison there is just a lack of it everywhere,' he said.\\n'It could take six months to a year, without a lot of help they could be on the streets for six months.\\n'When you think of the consequences of either being on the street, especially with the cold weather at the moment or you may have a roof over your head, sometimes there is only one choice.'\\nMr Stevens believes building more one-bedroom flats could help ease the problem.\\n'The average price is a hundred pounds a week to keep someone in a rented flat, prison is a lot more than that so I would imagine it would save the public purse quite a few pounds,' he said.\\nOfficial figures show 830 one-bedroom properties were built in the year to March 2016, of an overall total of 6,900 new properties in Wales.\\nMarc, 50, who has been in and out of prison for the past 20 years for burglary offences, said he struggled to find accommodation each time he was released.\\nHe said he would ask himself: 'Where am I going to stay? Where am I going to live? Have I got somewhere where I can see my daughter.'\\n'You\\'re put out among the same sort of people doing the same sort of thing, and it\\'s difficult, it\\'s difficult to get away from it. It\\'s like every man for himself, there\\'s nothing.'\\nMarc has now found stable accommodation with homeless charity Emmaus and said it had been life changing.\\n'You feel safe, you got hot food, you\\'ve got company of people in similar situations to yourself but all dealing with different issues. It\\'s a constructive, helpful atmosphere,' he said.\\nTom Clarke, chief executive of Emmaus South Wales, agreed there was not enough support available.\\n'We do still see [people] homeless on the streets, so clearly they haven\\'t got accommodation and haven\\'t got provision,' he said.\\n'I think the key is connecting people with the services they need. I don\\'t delude myself that Emmaus can offer a one size fits all for everyone, we can\\'t.\\n'But there must be other opportunities and given suitable encouragement I believe that can and should happen.'\\nA Welsh Government spokesman said the national pathway for homeless services to children, young people and adults in the secure estate had prevented many people from losing their home whilst serving their prison sentence.\\nIt added there were already significant demands for one-bedroom flats across the public and private sector and it was providing 20,000 new affordable homes in the next five years.\"])\n",+ "print('Predicted labels', y_pred)"+ ]+ }+ ],+ "metadata": {+ "interpreter": {+ "hash": "e9d36fc5b7c3dd4177ff1b60184dd696c0acc18150a44682abca4d769811bd46"+ },+ "kernelspec": {+ "display_name": "Python 3 (ipykernel)",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.8.12"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}\ No newline at end of filediff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.pyindex 1afb569eb..d64ced399 100644--- a/test/automl/test_notebook_example.py+++ b/test/automl/test_notebook_example.py@@ -4,12 +4,17 @@ from requests.exceptions import ChunkedEncodingErrordef test_automl(budget=5, dataset_format="dataframe", hpo_method=None):from flaml.data import load_openml_dataset+ import urllib3try:X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="test/", dataset_format=dataset_format)- except (OpenMLServerException, ChunkedEncodingError) as e:+ except (+ OpenMLServerException,+ ChunkedEncodingError,+ urllib3.exceptions.ReadTimeoutError,+ ) as e:print(e)return""" import AutoML class from flaml package """diff --git a/test/load_args.py b/test/load_args.pyindex 628886acb..3ad467c23 100644--- a/test/load_args.py+++ b/test/load_args.py@@ -1,7 +1,7 @@def test_load_args_sub():- from flaml.nlp.utils import HPOArgs+ from flaml.nlp.utils import HFArgs- HPOArgs.load_args()+ HFArgs.load_args()if __name__ == "__main__":diff --git a/test/nlp/test_autohf.py b/test/nlp/test_autohf.pyindex e882a78da..ba05024ff 100644--- a/test/nlp/test_autohf.py+++ b/test/nlp/test_autohf.py@@ -84,9 +84,10 @@ def test_hf_data():"task": "seq-classification","metric": "accuracy","log_file_name": "seqclass.log",+ "use_ray": False,}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "google/electra-small-discriminator","output_dir": "test/data/output/","ckpt_per_epoch": 5,@@ -116,7 +117,6 @@ def test_hf_data():pickle.dump(automl, f, pickle.HIGHEST_PROTOCOL)with open("automl.pkl", "rb") as f:automl = pickle.load(f)- shutil.rmtree("test/data/output/")automl.predict(X_test)automl.predict(["test test", "test test"])automl.predict(@@ -164,7 +164,7 @@ def _test_custom_data():"metric": "accuracy",}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "google/electra-small-discriminator","output_dir": "data/output/","ckpt_per_epoch": 1,@@ -183,6 +183,16 @@ def _test_custom_data():])+ import pickle++ automl.pickle("automl.pkl")++ with open("automl.pkl", "rb") as f:+ automl = pickle.load(f)+ config = automl.best_config.copy()+ config["learner"] = automl.best_estimator+ automl.trainable(config)+if __name__ == "__main__":test_hf_data()diff --git a/test/nlp/test_autohf_classificationhead.py b/test/nlp/test_autohf_classificationhead.pyindex 2a19e5129..8fc5fc474 100644--- a/test/nlp/test_autohf_classificationhead.py+++ b/test/nlp/test_autohf_classificationhead.py@@ -52,7 +52,7 @@ def test_classification_head():"metric": "accuracy",}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "google/electra-small-discriminator","output_dir": "test/data/output/","ckpt_per_epoch": 1,diff --git a/test/nlp/test_autohf_custom_metric.py b/test/nlp/test_autohf_custom_metric.pyindex a3dee305b..e81ffc78c 100644--- a/test/nlp/test_autohf_custom_metric.py+++ b/test/nlp/test_autohf_custom_metric.py@@ -19,8 +19,7 @@ def custom_metric(from flaml.model import TransformersEstimatorif estimator._trainer is None:- estimator._init_model_for_predict(X_test)- trainer = estimator._trainer+ trainer, _, _ = estimator._init_model_for_predict(X_test)estimator._trainer = Noneelse:trainer = estimator._trainer@@ -103,7 +102,7 @@ def test_custom_metric():"log_file_name": "seqclass.log",}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "google/electra-small-discriminator","output_dir": "data/output/","ckpt_per_epoch": 1,diff --git a/test/nlp/test_autohf_cv.py b/test/nlp/test_autohf_cv.pyindex 06aeb2b70..040bfc240 100644--- a/test/nlp/test_autohf_cv.py+++ b/test/nlp/test_autohf_cv.py@@ -43,7 +43,7 @@ def test_cv():"n_splits": 3,}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "google/electra-small-discriminator","output_dir": "test/data/output/","ckpt_per_epoch": 1,diff --git a/test/nlp/test_autohf_multichoice_classification.py b/test/nlp/test_autohf_multichoice_classification.pyindex 5d49809b0..b1b75db8e 100644--- a/test/nlp/test_autohf_multichoice_classification.py+++ b/test/nlp/test_autohf_multichoice_classification.py@@ -216,7 +216,7 @@ def test_mcc():"log_file_name": "seqclass.log",}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "google/electra-small-discriminator","output_dir": "test/data/output/","ckpt_per_epoch": 1,diff --git a/test/nlp/test_autohf_regression.py b/test/nlp/test_autohf_regression.pyindex 762091b01..2ccda05a6 100644--- a/test/nlp/test_autohf_regression.py+++ b/test/nlp/test_autohf_regression.py@@ -6,6 +6,9 @@ import pytestdef test_regression():try:import ray++ if not ray.is_initialized():+ ray.init()except ImportError:returnfrom flaml import AutoML@@ -65,10 +68,10 @@ def test_regression():"task": "seq-regression","metric": "pearsonr","starting_points": {"transformer": {"num_train_epochs": 1}},- "use_ray": True,+ "use_ray": {"local_dir": "data/outut/"},}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "google/electra-small-discriminator","output_dir": "test/data/output/","ckpt_per_epoch": 1,@@ -77,6 +80,7 @@ def test_regression():ray.shutdown()ray.init()+automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)diff --git a/test/nlp/test_autohf_summarization.py b/test/nlp/test_autohf_summarization.pyindex 690a85a62..b2af5e652 100644--- a/test/nlp/test_autohf_summarization.py+++ b/test/nlp/test_autohf_summarization.py@@ -58,7 +58,7 @@ def test_summarization():"log_file_name": "seqclass.log",}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "patrickvonplaten/t5-tiny-random","output_dir": "test/data/output/","ckpt_per_epoch": 1,diff --git a/test/nlp/test_autohf_tokenclassification.py b/test/nlp/test_autohf_tokenclassification.pyindex 41d1bb6b1..e47956a41 100644--- a/test/nlp/test_autohf_tokenclassification.py+++ b/test/nlp/test_autohf_tokenclassification.py@@ -726,7 +726,7 @@ def test_tokenclassification():"metric": "seqeval",}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "bert-base-uncased","output_dir": "test/data/output/","ckpt_per_epoch": 1,diff --git a/test/test_gpu.py b/test/test_gpu.pyindex 80bbcfc5f..e738dbbb4 100644--- a/test/test_gpu.py+++ b/test/test_gpu.py@@ -81,7 +81,7 @@ def _test_hf_data():"use_ray": True,}- automl_settings["custom_hpo_args"] = {+ automl_settings["hf_args"] = {"model_path": "facebook/muppet-roberta-base","output_dir": "test/data/output/","ckpt_per_epoch": 5,diff --git a/website/docs/Examples/AutoML-NLP.md b/website/docs/Examples/AutoML-NLP.mdindex a9e06ae0f..6d17b3f68 100644--- a/website/docs/Examples/AutoML-NLP.md+++ b/website/docs/Examples/AutoML-NLP.md@@ -26,8 +26,8 @@ automl = AutoML()automl_settings = {"time_budget": 100,"task": "seq-classification",- "custom_hpo_args": {"output_dir": "data/output/"},- "gpu_per_trial": 1, # set to 0 if no GPU is available+ "hf_args": {"output_dir": "data/output/"}, # setting the huggingface arguments: output directory+ "gpu_per_trial": 1, # set to 0 if no GPU is available}automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)automl.predict(X_test)@@ -77,11 +77,11 @@ automl_settings = {"task": "seq-regression","metric": "rmse",}-automl_settings["custom_hpo_args"] = {- "model_path": "google/electra-small-discriminator",- "output_dir": "data/output/",- "ckpt_per_epoch": 5,- "fp16": False,+automl_settings["hf_args"] = { # setting the huggingface arguments+ "model_path": "google/electra-small-discriminator", # setting the language model+ "output_dir": "data/output/", # setting the output directory+ "ckpt_per_epoch": 5, # setting the number of checkpoints per epoch+ "fp16": False, # setting whether to use FP16}automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings@@ -127,11 +127,11 @@ automl_settings = {"task": "summarization","metric": "rouge1",}-automl_settings["custom_hpo_args"] = {- "model_path": "t5-small",- "output_dir": "data/output/",- "ckpt_per_epoch": 5,- "fp16": False,+automl_settings["hf_args"] = { # setting the huggingface arguments+ "model_path": "t5-small", # setting the language model+ "output_dir": "data/output/", # setting the output directory+ "ckpt_per_epoch": 5, # setting the number of checkpoints per epoch+ "fp16": False, # setting whether to use FP16}automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings@@ -205,4 +205,10 @@ Model config T5Config {}```-For tasks that are not currently supported, use `flaml.tune` for [customized tuning](Tune-HuggingFace).\ No newline at end of file+For tasks that are not currently supported, use `flaml.tune` for [customized tuning](Tune-HuggingFace).++### Link to Jupyter notebook++To run these examples in our Jupyter notebook, please go to:++[Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/automl_nlp.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/automl_nlp.ipynb)\ No newline at end of file