A programming framework for agentic AI
tune api for schedulers (#322)
Qingyun Wu17b17d084ff1
diff --git a/flaml/automl.py b/flaml/automl.pyindex 91eaf2215..d6f23ce1a 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -1546,11 +1546,12 @@ class AutoML(BaseEstimator):return points@property- def prune_attr(self) -> Optional[str]:- """Attribute for pruning+ def resource_attr(self) -> Optional[str]:+ """Attribute of the resource dimension.Returns:- A string for the sample size attribute or None+ A string for the sample size attribute+ (the resource attribute in AutoML) or None."""return "FLAML_sample_size" if self._sample else None@@ -2178,7 +2179,7 @@ class AutoML(BaseEstimator):low_cost_partial_config=self.low_cost_partial_config,points_to_evaluate=self.points_to_evaluate,cat_hp_cost=self.cat_hp_cost,- prune_attr=self.prune_attr,+ resource_attr=self.resource_attr,min_resource=self.min_resource,max_resource=self.max_resource,config_constraints=[@@ -2326,11 +2327,11 @@ class AutoML(BaseEstimator):)search_space = search_state.search_spaceif self._sample:- prune_attr = "FLAML_sample_size"+ resource_attr = "FLAML_sample_size"min_resource = self._min_sample_sizemax_resource = self._state.data_size[0]else:- prune_attr = min_resource = max_resource = None+ resource_attr = min_resource = max_resource = Nonelearner_class = self._state.learner_classes.get(estimator)if "grid" == self._hpo_method: # for synthetic exp onlypoints_to_evaluate = []@@ -2362,7 +2363,7 @@ class AutoML(BaseEstimator):points_to_evaluate=points_to_evaluate,low_cost_partial_config=low_cost_partial_config,cat_hp_cost=search_state.cat_hp_cost,- prune_attr=prune_attr,+ resource_attr=resource_attr,min_resource=min_resource,max_resource=max_resource,config_constraints=[diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex 147d1c055..9de56c203 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -45,7 +45,7 @@ class BlendSearch(Searcher):evaluated_rewards: Optional[List] = None,time_budget_s: Union[int, float] = None,num_samples: Optional[int] = None,- prune_attr: Optional[str] = None,+ resource_attr: Optional[str] = None,min_resource: Optional[float] = None,max_resource: Optional[float] = None,reduction_factor: Optional[float] = None,@@ -91,17 +91,10 @@ class BlendSearch(Searcher):points_to_evaluate.time_budget_s: int or float | Time budget in seconds.num_samples: int | The number of configs to try.- prune_attr: A string of the attribute used for pruning.- Not necessarily in space.- When prune_attr is in space, it is a hyperparameter, e.g.,- 'n_iters', and the best value is unknown.- When prune_attr is not in space, it is a resource dimension,- e.g., 'sample_size', and the peak performance is assumed- to be at the max_resource.- min_resource: A float of the minimal resource to use for the- prune_attr; only valid if prune_attr is not in space.- max_resource: A float of the maximal resource to use for the- prune_attr; only valid if prune_attr is not in space.+ resource_attr: A string to specify the resource dimension and the best+ performance is assumed to be at the max_resource.+ min_resource: A float of the minimal resource to use for the resource_attr.+ max_resource: A float of the maximal resource to use for the resource_attr.reduction_factor: A float of the reduction factor used forincremental pruning.global_search_alg: A Searcher instance as the global search@@ -160,7 +153,7 @@ class BlendSearch(Searcher):metric,mode,space,- prune_attr,+ resource_attr,min_resource,max_resource,reduction_factor,@@ -409,7 +402,7 @@ class BlendSearch(Searcher):if (objective - self._metric_target) * self._ls.metric_op < 0:self._metric_target = objectiveif self._ls.resource:- self._best_resource = config[self._ls.prune_attr]+ self._best_resource = config[self._ls.resource_attr]if thread_id:if not self._metric_constraint_satisfied:# no point has been found to satisfy metric constraint@@ -637,7 +630,7 @@ class BlendSearch(Searcher):# return Noneconfig = self._search_thread_pool[choice].suggest(trial_id)if not choice and config is not None and self._ls.resource:- config[self._ls.prune_attr] = self.best_resource+ config[self._ls.resource_attr] = self.best_resourceelif choice and config is None:# local search thread finishesif self._search_thread_pool[choice].converged:@@ -975,7 +968,7 @@ class BlendSearchTuner(BlendSearch, NNITuner):self._ls.metric,self._mode,config,- self._ls.prune_attr,+ self._ls.resource_attr,self._ls.min_resource,self._ls.max_resource,self._ls.resource_multiple_factor,diff --git a/flaml/searcher/flow2.py b/flaml/searcher/flow2.pyindex b881dadc4..b1e69c9b8 100644--- a/flaml/searcher/flow2.py+++ b/flaml/searcher/flow2.py@@ -39,7 +39,7 @@ class FLOW2(Searcher):metric: Optional[str] = None,mode: Optional[str] = None,space: Optional[dict] = None,- prune_attr: Optional[str] = None,+ resource_attr: Optional[str] = None,min_resource: Optional[float] = None,max_resource: Optional[float] = None,resource_multiple_factor: Optional[float] = 4,@@ -67,17 +67,10 @@ class FLOW2(Searcher):i.e., the relative cost of thethree choices of 'tree_method' is 1, 1 and 2 respectively.space: A dictionary to specify the search space.- prune_attr: A string of the attribute used for pruning.- Not necessarily in space.- When prune_attr is in space, it is a hyperparameter, e.g.,- 'n_iters', and the best value is unknown.- When prune_attr is not in space, it is a resource dimension,- e.g., 'sample_size', and the peak performance is assumed- to be at the max_resource.- min_resource: A float of the minimal resource to use for the- prune_attr; only valid if prune_attr is not in space.- max_resource: A float of the maximal resource to use for the- prune_attr; only valid if prune_attr is not in space.+ resource_attr: A string to specify the resource dimension and the best+ performance is assumed to be at the max_resource.+ min_resource: A float of the minimal resource to use for the resource_attr.+ max_resource: A float of the maximal resource to use for the resource_attr.resource_multiple_factor: A float of the multiplicative factorused for increasing resource.cost_attr: A string of the attribute used for cost.@@ -100,7 +93,7 @@ class FLOW2(Searcher):self.seed = seedself.init_config = init_configself.best_config = flatten_dict(init_config)- self.prune_attr = prune_attr+ self.resource_attr = resource_attrself.min_resource = min_resourceself.resource_multiple_factor = resource_multiple_factor or 4self.cost_attr = cost_attr@@ -148,11 +141,15 @@ class FLOW2(Searcher):if not hier:self._space_keys = sorted(self._tunable_keys)self.hierarchical = hier- if self.prune_attr and self.prune_attr not in self._space and self.max_resource:+ if (+ self.resource_attr+ and self.resource_attr not in self._space+ and self.max_resource+ ):self.min_resource = self.min_resource or self._min_resource()self._resource = self._round(self.min_resource)if not hier:- self._space_keys.append(self.prune_attr)+ self._space_keys.append(self.resource_attr)else:self._resource = Noneself.incumbent = {}@@ -252,7 +249,7 @@ class FLOW2(Searcher):if partial_config == self.init_config:self._reset_times += 1if self._resource:- config[self.prune_attr] = self.min_resource+ config[self.resource_attr] = self.min_resourcereturn config, spacedef create(@@ -264,7 +261,7 @@ class FLOW2(Searcher):self.metric,self.mode,space,- self.prune_attr,+ self.resource_attr,self.min_resource,self.max_resource,self.resource_multiple_factor,@@ -328,7 +325,7 @@ class FLOW2(Searcher):self.incumbent = self.normalize(self.best_config)self.cost_incumbent = result.get(self.cost_attr)if self._resource:- self._resource = self.best_config[self.prune_attr]+ self._resource = self.best_config[self.resource_attr]self._num_complete4incumbent = 0self._cost_complete4incumbent = 0self._num_proposedby_incumbent = 0@@ -377,7 +374,7 @@ class FLOW2(Searcher):if self.best_config != config:self.best_config = configif self._resource:- self._resource = config[self.prune_attr]+ self._resource = config[self.resource_attr]self.incumbent = self.normalize(self.best_config)self.cost_incumbent = result.get(self.cost_attr)self._cost_complete4incumbent = 0@@ -495,18 +492,18 @@ class FLOW2(Searcher):self._resource = self._round(self._resource * self.resource_multiple_factor)self.cost_incumbent *= self._resource / old_resourceconfig = self.best_config.copy()- config[self.prune_attr] = self._resource+ config[self.resource_attr] = self._resourceself._direction_tried = Noneself._configs[trial_id] = (config, self.step)return unflatten_dict(config)def _project(self, config):- """project normalized config in the feasible region and set prune_attr"""+ """project normalized config in the feasible region and set resource_attr"""for key in self._bounded_keys:value = config[key]config[key] = max(0, min(1, value))if self._resource:- config[self.prune_attr] = self._resource+ config[self.resource_attr] = self._resource@propertydef can_suggest(self) -> bool:@@ -525,7 +522,7 @@ class FLOW2(Searcher):keys = sorted(config.keys()) if self.hierarchical else self._space_keysfor key in keys:value = config[key]- if key == self.prune_attr:+ if key == self.resource_attr:value_list.append(value)else:# key must be in space@@ -556,7 +553,7 @@ class FLOW2(Searcher):"""whether the incumbent can reach the incumbent of other."""config1, config2 = self.best_config, other.best_configincumbent1, incumbent2 = self.incumbent, other.incumbent- if self._resource and config1[self.prune_attr] > config2[self.prune_attr]:+ if self._resource and config1[self.resource_attr] > config2[self.resource_attr]:# resource will not decreasereturn Falsefor key in self._unordered_cat_hp:diff --git a/flaml/tune/space.py b/flaml/tune/space.pyindex 91fe08868..d050eed1a 100644--- a/flaml/tune/space.py+++ b/flaml/tune/space.py@@ -247,7 +247,7 @@ def normalize(config_norm = {}for key, value in config.items():domain = space.get(key)- if domain is None: # e.g., prune_attr+ if domain is None: # e.g., resource_attrconfig_norm[key] = valuecontinueif not callable(getattr(domain, "get_sampler", None)):@@ -405,7 +405,7 @@ def denormalize(# Handle int (4.6 -> 5)if isinstance(domain, sample.Integer):config_denorm[key] = int(round(config_denorm[key]))- else: # prune_attr+ else: # resource_attrconfig_denorm[key] = valuereturn config_denormdiff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex c98c3bc6b..b7726453a 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -17,6 +17,7 @@ try:except (ImportError, AssertionError):ray_import = Falsefrom .analysis import ExperimentAnalysis as EA+from .result import DEFAULT_METRICimport logging@@ -117,11 +118,11 @@ def run(time_budget_s: Union[int, float] = None,points_to_evaluate: Optional[List[dict]] = None,evaluated_rewards: Optional[List] = None,- prune_attr: Optional[str] = None,+ resource_attr: Optional[str] = None,min_resource: Optional[float] = None,max_resource: Optional[float] = None,reduction_factor: Optional[float] = None,- report_intermediate_result: Optional[bool] = False,+ scheduler: Optional = None,search_alg=None,verbose: Optional[int] = 2,local_dir: Optional[str] = None,@@ -205,21 +206,29 @@ def run(points_to_evaluate are 3.0 and 1.0 respectively and want toinform run()- prune_attr: A string of the attribute used for pruning.- Not necessarily in space.- When prune_attr is in space, it is a hyperparameter, e.g.,- 'n_iters', and the best value is unknown.- When prune_attr is not in space, it is a resource dimension,- e.g., 'sample_size', and the peak performance is assumed- to be at the max_resource.- min_resource: A float of the minimal resource to use for the- prune_attr; only valid if prune_attr is not in space.- max_resource: A float of the maximal resource to use for the- prune_attr; only valid if prune_attr is not in space.+ resource_attr: A string to specify the resource dimension used by+ the scheduler via "scheduler".+ min_resource: A float of the minimal resource to use for the resource_attr.+ max_resource: A float of the maximal resource to use for the resource_attr.reduction_factor: A float of the reduction factor used for incrementalpruning.- report_intermediate_result: A boolean of whether intermediate results- are reported. If so, early stopping and pruning can be used.+ scheduler: A scheduler for executing the experiment. Can be None, 'flaml',+ 'asha' or a custom instance of the TrialScheduler class. Default is None:+ in this case when resource_attr is provided, the 'flaml' scheduler will be+ used, otherwise no scheduler will be used. When set 'flaml', an+ authentic scheduler implemented in FLAML will be used. It does not+ require users to report intermediate results in training_function.+ Find more details abuot this scheduler in this paper+ https://arxiv.org/pdf/1911.04706.pdf).+ When set 'asha', the input for arguments "resource_attr",+ "min_resource", "max_resource" and "reduction_factor" will be passed+ to ASHA's "time_attr", "max_t", "grace_period" and "reduction_factor"+ respectively. You can also provide a self-defined scheduler instance+ of the TrialScheduler class. When 'asha' or self-defined scheduler is+ used, you usually need to report intermediate results in the training+ function. Please find examples using different types of schedulers+ and how to set up the corresponding training functions in+ test/tune/test_scheduler.py. TODO: point to notebook examples.search_alg: An instance of BlendSearch as the search algorithmto be used. The same instance can be used for iterative tuning.e.g.,@@ -295,6 +304,20 @@ def run(from ..searcher.blendsearch import BlendSearchif search_alg is None:+ flaml_scheduler_resource_attr = (+ flaml_scheduler_min_resource+ ) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None+ if scheduler in (None, "flaml"):++ # when scheduler is set 'flaml', we will use a scheduler that is+ # authentic to the search algorithms in flaml. After setting up+ # the search algorithm accordingly, we need to set scheduler to+ # None in case it is later used in the trial runner.+ flaml_scheduler_resource_attr = resource_attr+ flaml_scheduler_min_resource = min_resource+ flaml_scheduler_max_resource = max_resource+ flaml_scheduler_reduction_factor = reduction_factor+ scheduler = Nonesearch_alg = BlendSearch(metric=metric or DEFAULT_METRIC,mode=mode,@@ -305,10 +328,10 @@ def run(cat_hp_cost=cat_hp_cost,time_budget_s=time_budget_s,num_samples=num_samples,- prune_attr=prune_attr,- min_resource=min_resource,- max_resource=max_resource,- reduction_factor=reduction_factor,+ resource_attr=flaml_scheduler_resource_attr,+ min_resource=flaml_scheduler_min_resource,+ max_resource=flaml_scheduler_max_resource,+ reduction_factor=flaml_scheduler_reduction_factor,config_constraints=config_constraints,metric_constraints=metric_constraints,)@@ -334,12 +357,11 @@ def run(searcher.set_search_properties(metric, mode, config, setting)else:searcher.set_search_properties(metric, mode, config)- scheduler = None- if report_intermediate_result:+ if scheduler == "asha":params = {}- # scheduler resource_dimension=prune_attr- if prune_attr:- params["time_attr"] = prune_attr+ # scheduler resource_dimension=resource_attr+ if resource_attr:+ params["time_attr"] = resource_attrif max_resource:params["max_t"] = max_resourceif min_resource:diff --git a/flaml/version.py b/flaml/version.pyindex deded3247..3e2f46a3a 100644--- a/flaml/version.py+++ b/flaml/version.py@@ -1 +1 @@-__version__ = "0.8.2"+__version__ = "0.9.0"diff --git a/notebook/flaml_finetune_transformer.ipynb b/notebook/flaml_finetune_transformer.ipynbindex 01718b9cc..beae8aed3 100644--- a/notebook/flaml_finetune_transformer.ipynb+++ b/notebook/flaml_finetune_transformer.ipynb@@ -29,7 +29,7 @@},{"cell_type": "code",- "execution_count": 2,+ "execution_count": 1,"metadata": {},"outputs": [],"source": [@@ -38,7 +38,7 @@},{"cell_type": "code",- "execution_count": 3,+ "execution_count": 2,"metadata": {},"outputs": [],"source": [@@ -47,7 +47,7 @@},{"cell_type": "code",- "execution_count": 4,+ "execution_count": 3,"metadata": {},"outputs": [],"source": [@@ -56,18 +56,18 @@},{"cell_type": "code",- "execution_count": 5,+ "execution_count": 4,"metadata": {},"outputs": [{- "output_type": "execute_result","data": {"text/plain": ["{'input_ids': [101, 2023, 2003, 1037, 3231, 102], 'attention_mask': [1, 1, 1, 1, 1, 1]}"]},+ "execution_count": 4,"metadata": {},- "execution_count": 5+ "output_type": "execute_result"}],"source": [@@ -83,7 +83,7 @@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": 5,"metadata": {},"outputs": [],"source": [@@ -92,7 +92,7 @@},{"cell_type": "code",- "execution_count": 7,+ "execution_count": 6,"metadata": {},"outputs": [],"source": [@@ -101,14 +101,14 @@},{"cell_type": "code",- "execution_count": 8,+ "execution_count": 7,"metadata": {},"outputs": [{- "output_type": "stream","name": "stderr",+ "output_type": "stream","text": [- "Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"+ "Reusing dataset glue (/home/ec2-user/.cache/huggingface/datasets/glue/cola/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4)\n"]}],@@ -118,7 +118,7 @@},{"cell_type": "code",- "execution_count": 9,+ "execution_count": 8,"metadata": {},"outputs": [],"source": [@@ -130,62 +130,68 @@},{"cell_type": "code",- "execution_count": 10,+ "execution_count": 9,"metadata": {},"outputs": [{- "output_type": "display_data","data": {- "text/plain": "HBox(children=(FloatProgress(value=0.0, max=9.0), HTML(value='')))","application/vnd.jupyter.widget-view+json": {+ "model_id": "0dcf9ca8ce024a2b832606a6a3219b17","version_major": 2,- "version_minor": 0,- "model_id": "ecc66e6795f848e0a41e6cf1ce37bdf2"- }+ "version_minor": 0+ },+ "text/plain": [+ "HBox(children=(FloatProgress(value=0.0, max=9.0), HTML(value='')))"+ ]},- "metadata": {}+ "metadata": {},+ "output_type": "display_data"},{- "output_type": "stream","name": "stdout",+ "output_type": "stream","text": ["\n"]},{- "output_type": "display_data","data": {- "text/plain": "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))","application/vnd.jupyter.widget-view+json": {+ "model_id": "c58845729f0a4261830ad679891e7c77","version_major": 2,- "version_minor": 0,- "model_id": "2d33fc70b80b403080ad8c0e77ed1891"- }+ "version_minor": 0+ },+ "text/plain": [+ "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"+ ]},- "metadata": {}+ "metadata": {},+ "output_type": "display_data"},{- "output_type": "stream","name": "stdout",+ "output_type": "stream","text": ["\n"]},{- "output_type": "display_data","data": {- "text/plain": "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))","application/vnd.jupyter.widget-view+json": {+ "model_id": "9716d177a40748008cc6089e3d52a1d5","version_major": 2,- "version_minor": 0,- "model_id": "d2ab3feb1a354187abb2dded0ead404f"- }+ "version_minor": 0+ },+ "text/plain": [+ "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"+ ]},- "metadata": {}+ "metadata": {},+ "output_type": "display_data"},{- "output_type": "stream","name": "stdout",+ "output_type": "stream","text": ["\n"]@@ -197,11 +203,10 @@},{"cell_type": "code",- "execution_count": 11,+ "execution_count": 10,"metadata": {},"outputs": [{- "output_type": "execute_result","data": {"text/plain": ["{'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",@@ -229,8 +234,9 @@" 'sentence': \"Our friends won't buy this analysis, let alone the next one we propose.\"}"]},+ "execution_count": 10,"metadata": {},- "execution_count": 11+ "output_type": "execute_result"}],"source": [@@ -246,7 +252,7 @@},{"cell_type": "code",- "execution_count": 12,+ "execution_count": 11,"metadata": {},"outputs": [],"source": [@@ -255,12 +261,12 @@},{"cell_type": "code",- "execution_count": 13,+ "execution_count": 12,"metadata": {},"outputs": [{- "output_type": "stream","name": "stderr",+ "output_type": "stream","text": ["Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n","- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",@@ -277,11 +283,10 @@},{"cell_type": "code",- "execution_count": 14,+ "execution_count": 13,"metadata": {},"outputs": [{- "output_type": "execute_result","data": {"text/plain": ["DistilBertForSequenceClassification(\n",@@ -399,8 +404,9 @@")"]},+ "execution_count": 13,"metadata": {},- "execution_count": 14+ "output_type": "execute_result"}],"source": [@@ -416,7 +422,7 @@},{"cell_type": "code",- "execution_count": 15,+ "execution_count": 14,"metadata": {},"outputs": [],"source": [@@ -425,11 +431,10 @@},{"cell_type": "code",- "execution_count": 16,+ "execution_count": 15,"metadata": {},"outputs": [{- "output_type": "execute_result","data": {"text/plain": ["Metric(name: \"glue\", features: {'predictions': Value(dtype='int64', id=None), 'references': Value(dtype='int64', id=None)}, usage: \"\"\"\n",@@ -477,8 +482,9 @@"\"\"\", stored examples: 0)"]},+ "execution_count": 15,"metadata": {},- "execution_count": 16+ "output_type": "execute_result"}],"source": [@@ -487,7 +493,7 @@},{"cell_type": "code",- "execution_count": 17,+ "execution_count": 16,"metadata": {},"outputs": [],"source": [@@ -507,7 +513,7 @@},{"cell_type": "code",- "execution_count": 18,+ "execution_count": 17,"metadata": {},"outputs": [],"source": [@@ -517,7 +523,7 @@},{"cell_type": "code",- "execution_count": 19,+ "execution_count": 18,"metadata": {},"outputs": [],"source": [@@ -529,7 +535,7 @@},{"cell_type": "code",- "execution_count": 20,+ "execution_count": 19,"metadata": {},"outputs": [],"source": [@@ -545,40 +551,70 @@},{"cell_type": "code",- "execution_count": 21,+ "execution_count": 20,"metadata": {},"outputs": [{+ "name": "stdout","output_type": "stream",- "name": "stderr",- "text": [- "/home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "\n <div>\n <style>\n /* Turns off some styling */\n progress {\n /* gets rid of default border in Firefox and Opera. */\n border: none;\n /* Needs to be in here for Safari polyfill so background images work as expected. */\n background-size: auto;\n }\n </style>\n \n <progress value='2' max='804' style='width:300px; height:20px; vertical-align: middle;'></progress>\n [ 2/804 : < :, Epoch 0.00/3]\n </div>\n <table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: left;\">\n <th>Step</th>\n <th>Training Loss</th>\n </tr>\n </thead>\n <tbody>\n </tbody>\n</table><p>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stderr","text": [- "/home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n"+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"]},{- "output_type": "execute_result","data": {+ "text/html": [+ "\n",+ " <div>\n",+ " <style>\n",+ " /* Turns off some styling */\n",+ " progress {\n",+ " /* gets rid of default border in Firefox and Opera. */\n",+ " border: none;\n",+ " /* Needs to be in here for Safari polyfill so background images work as expected. */\n",+ " background-size: auto;\n",+ " }\n",+ " </style>\n",+ " \n",+ " <progress value='1591' max='3207' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",+ " [1591/3207 1:03:06 < 1:04:11, 0.42 it/s, Epoch 1.49/3]\n",+ " </div>\n",+ " <table border=\"1\" class=\"dataframe\">\n",+ " <thead>\n",+ " <tr style=\"text-align: left;\">\n",+ " <th>Step</th>\n",+ " <th>Training Loss</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <td>500</td>\n",+ " <td>0.571000</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <td>1000</td>\n",+ " <td>0.515400</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <td>1500</td>\n",+ " <td>0.356100</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table><p>"+ ],"text/plain": [- "TrainOutput(global_step=804, training_loss=0.3209413462017306, metrics={'train_runtime': 115.5328, 'train_samples_per_second': 6.959, 'total_flos': 238363718990580.0, 'epoch': 3.0, 'init_mem_cpu_alloc_delta': 2336600064, 'init_mem_gpu_alloc_delta': 268953088, 'init_mem_cpu_peaked_delta': 257929216, 'init_mem_gpu_peaked_delta': 0, 'train_mem_cpu_alloc_delta': 2381066240, 'train_mem_gpu_alloc_delta': 806788096, 'train_mem_cpu_peaked_delta': 186974208, 'train_mem_gpu_peaked_delta': 550790144})"+ "<IPython.core.display.HTML object>"]},"metadata": {},- "execution_count": 21+ "output_type": "display_data"}],"source": [@@ -586,6 +622,8 @@]},{+ "cell_type": "markdown",+ "metadata": {},"source": ["## Hyperparameter Optimization\n","\n",@@ -595,13 +633,11 @@"### Step 1. Define training method\n","\n","We define a function `train_distilbert(config: dict)` that accepts a hyperparameter configuration dict `config`. The specific configs will be generated by flaml's search algorithm in a given search space.\n"- ],- "cell_type": "markdown",- "metadata": {}+ ]},{"cell_type": "code",- "execution_count": 22,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -656,6 +692,8 @@]},{+ "cell_type": "markdown",+ "metadata": {},"source": ["### Step 2. Define the search\n","\n",@@ -664,13 +702,11 @@"- The `search_space` for our hyperparameters\n","- The metric and the mode ('max' or 'min') for optimization\n","- The constraints (`n_cpus`, `n_gpus`, `num_samples`, and `time_budget_s`)"- ],- "cell_type": "markdown",- "metadata": {}+ ]},{"cell_type": "code",- "execution_count": 23,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -687,7 +723,7 @@},{"cell_type": "code",- "execution_count": 24,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -704,809 +740,141 @@]},{+ "cell_type": "markdown",+ "metadata": {},"source": ["### Step 3. Launch with `flaml.tune.run`\n","\n","We are now ready to launch the tuning using `flaml.tune.run`:"- ],- "cell_type": "markdown",- "metadata": {}+ ]},{"cell_type": "code",- "execution_count": 25,+ "execution_count": null,"metadata": {},"outputs": [{- "output_type": "stream",- "name": "stderr",- "text": [- "2021-05-07 02:35:57,130\tINFO services.py:1172 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n",- "2021-05-07 02:35:58,044\tWARNING function_runner.py:540 -- Function checkpointing is disabled. This may result in unexpected behavior when using checkpointing features or certain schedulers. To enable, set the train function arguments to be `func(config, checkpoint_dir=None)`.\n",- "Tuning started...\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 26.0/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 1/infinite (1 RUNNING)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stderr",- "text": [- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 62.07ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 40.87ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 107.60ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 105.70ba/s]\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m {'train_runtime': 45.5778, 'train_samples_per_second': 5.88, 'epoch': 1.0}\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m {'eval_loss': 0.5879864692687988, 'eval_matthews_correlation': 0.0, 'eval_runtime': 1.7063, 'eval_samples_per_second': 611.265, 'epoch': 1.0}\n",- "Trial train_distilbert_a0c303d0 reported loss=0.5879864692687988,matthews_correlation=0.0 with parameters={'num_train_epochs': 1, 'learning_rate': 5.61151641533451e-06, 'adam_epsilon': 7.969454818643929e-08, 'adam_beta1': 0.9390788489441669, 'adam_beta2': 0.99186521389353}.\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",- "\u001b[2m\u001b[36m(pid=886303)\u001b[0m mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 30.9/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 2/infinite (1 PENDING, 1 RUNNING)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream","name": "stdout",- "text": [- "Trial train_distilbert_a0c303d0 completed. Last result: loss=0.5879864692687988,matthews_correlation=0.0\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 61.83ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 41.19ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 108.40ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 104.85ba/s]\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m {'train_runtime': 62.1006, 'train_samples_per_second': 6.248, 'epoch': 1.45}\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m {'eval_loss': 0.6030182838439941, 'eval_matthews_correlation': 0.0, 'eval_runtime': 1.7026, 'eval_samples_per_second': 612.584, 'epoch': 1.45}\n",- "Trial train_distilbert_a0c303d1 reported loss=0.6030182838439941,matthews_correlation=0.0 with parameters={'num_train_epochs': 1.444265389543504, 'learning_rate': 2.051338263087453e-06, 'adam_epsilon': 2.0511104188434023e-09, 'adam_beta1': 0.8110358863119579, 'adam_beta2': 0.997213662958137}.\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",- "\u001b[2m\u001b[36m(pid=886302)\u001b[0m mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.2/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 3/infinite (1 PENDING, 1 RUNNING, 1 TERMINATED)<br><br>"- },- "metadata": {}- },- {"output_type": "stream",- "name": "stdout","text": [- "Trial train_distilbert_a0c303d1 completed. Last result: loss=0.6030182838439941,matthews_correlation=0.0\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 56.45ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 39.00ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 112.51ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 106.76ba/s]\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m {'train_runtime': 44.0366, 'train_samples_per_second': 6.086, 'epoch': 1.0}\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m {'eval_loss': 0.5865175724029541, 'eval_matthews_correlation': 0.0, 'eval_runtime': 1.6974, 'eval_samples_per_second': 614.462, 'epoch': 1.0}\n",- "Trial train_distilbert_c39b2ef0 reported loss=0.5865175724029541,matthews_correlation=0.0 with parameters={'num_train_epochs': 1.0, 'learning_rate': 5.265428651017862e-06, 'adam_epsilon': 1e-07, 'adam_beta1': 0.9093950363089345, 'adam_beta2': 0.9937145453421068}.\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",- "\u001b[2m\u001b[36m(pid=886305)\u001b[0m mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"]},{- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.4/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 4/infinite (1 PENDING, 1 RUNNING, 2 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_c39b2ef0 completed. Last result: loss=0.5865175724029541,matthews_correlation=0.0\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 59.27ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 40.35ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 114.16ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 92.98ba/s]\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m {'train_runtime': 47.2831, 'train_samples_per_second': 5.837, 'epoch': 1.03}\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m {'eval_loss': 0.5813134908676147, 'eval_matthews_correlation': 0.0, 'eval_runtime': 1.7102, 'eval_samples_per_second': 609.872, 'epoch': 1.03}\n",- "Trial train_distilbert_f00776e2 reported loss=0.5813134908676147,matthews_correlation=0.0 with parameters={'num_train_epochs': 1.027230096840913, 'learning_rate': 5.980351945986672e-06, 'adam_epsilon': 4.3894312769297216e-08, 'adam_beta1': 0.9687626615793994, 'adam_beta2': 0.9900193241041526}.\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",- "\u001b[2m\u001b[36m(pid=886304)\u001b[0m mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.7/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 5/infinite (1 PENDING, 1 RUNNING, 3 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_f00776e2 completed. Last result: loss=0.5813134908676147,matthews_correlation=0.0\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 58.13ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 39.40ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 92.35ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 106.15ba/s]\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m {'train_runtime': 44.3622, 'train_samples_per_second': 6.041, 'epoch': 1.0}\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m {'eval_loss': 0.5855756998062134, 'eval_matthews_correlation': 0.0, 'eval_runtime': 1.6326, 'eval_samples_per_second': 638.84, 'epoch': 1.0}\n",- "Trial train_distilbert_11ab3900 reported loss=0.5855756998062134,matthews_correlation=0.0 with parameters={'num_train_epochs': 1.0, 'learning_rate': 5.066078755222997e-06, 'adam_epsilon': 7.092964136440028e-08, 'adam_beta1': 0.9621979585561743, 'adam_beta2': 0.9918380608681451}.\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",- "\u001b[2m\u001b[36m(pid=892770)\u001b[0m mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 32.0/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 6/infinite (1 PENDING, 1 RUNNING, 4 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_11ab3900 completed. Last result: loss=0.5855756998062134,matthews_correlation=0.0\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 55.93ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 40.18ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 104.47ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 102.67ba/s]\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m {'train_runtime': 88.0772, 'train_samples_per_second': 6.562, 'epoch': 2.16}\n",- "\u001b[2m\u001b[36m(pid=897725)\u001b[0m {'eval_loss': 0.5316324830055237, 'eval_matthews_correlation': 0.38889272875750597, 'eval_runtime': 1.6116, 'eval_samples_per_second': 647.165, 'epoch': 2.16}\n",- "Trial train_distilbert_353025b6 reported loss=0.5316324830055237,matthews_correlation=0.38889272875750597 with parameters={'num_train_epochs': 2.1544304289135847, 'learning_rate': 6.215678437115527e-06, 'adam_epsilon': 8.954255073716448e-08, 'adam_beta1': 0.9159597393321596, 'adam_beta2': 0.9918923676622686}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 30.9/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 7/infinite (1 PENDING, 1 RUNNING, 5 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_353025b6 completed. Last result: loss=0.5316324830055237,matthews_correlation=0.38889272875750597\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 60.41ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 40.27ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 107.10ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 93.66ba/s]\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m {'train_runtime': 45.7159, 'train_samples_per_second': 5.862, 'epoch': 1.0}\n",- "\u001b[2m\u001b[36m(pid=907288)\u001b[0m {'eval_loss': 0.5385054349899292, 'eval_matthews_correlation': 0.2805581766595423, 'eval_runtime': 1.6966, 'eval_samples_per_second': 614.762, 'epoch': 1.0}\n",- "Trial train_distilbert_5728a1de reported loss=0.5385054349899292,matthews_correlation=0.2805581766595423 with parameters={'num_train_epochs': 1.0, 'learning_rate': 1.0090242363457245e-05, 'adam_epsilon': 1e-07, 'adam_beta1': 0.9269328215628503, 'adam_beta2': 0.9931456651827125}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.3/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 8/infinite (1 PENDING, 1 RUNNING, 6 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_5728a1de completed. Last result: loss=0.5385054349899292,matthews_correlation=0.2805581766595423\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 60.71ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 40.09ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 96.21ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 89.91ba/s]\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m {'train_runtime': 111.6359, 'train_samples_per_second': 6.557, 'epoch': 2.73}\n",- "\u001b[2m\u001b[36m(pid=908756)\u001b[0m {'eval_loss': 0.5391769409179688, 'eval_matthews_correlation': 0.3272948213494272, 'eval_runtime': 1.7214, 'eval_samples_per_second': 605.887, 'epoch': 2.73}\n",- "Trial train_distilbert_9394c2e2 reported loss=0.5391769409179688,matthews_correlation=0.3272948213494272 with parameters={'num_train_epochs': 2.729346084540195, 'learning_rate': 3.456743686220407e-06, 'adam_epsilon': 4.499745343526232e-08, 'adam_beta1': 0.9281057667134762, 'adam_beta2': 0.9906135322351715}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.6/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 9/infinite (1 PENDING, 1 RUNNING, 7 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_9394c2e2 completed. Last result: loss=0.5391769409179688,matthews_correlation=0.3272948213494272\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 67.17ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 43.92ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 92.79ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 113.54ba/s]\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m {'train_runtime': 66.0391, 'train_samples_per_second': 6.481, 'epoch': 1.6}\n",- "\u001b[2m\u001b[36m(pid=912284)\u001b[0m {'eval_loss': 0.5275164842605591, 'eval_matthews_correlation': 0.37917684067701946, 'eval_runtime': 1.8839, 'eval_samples_per_second': 553.629, 'epoch': 1.6}\n",- "Trial train_distilbert_b6543fec reported loss=0.5275164842605591,matthews_correlation=0.37917684067701946 with parameters={'num_train_epochs': 1.5953752206236405, 'learning_rate': 7.011758405605033e-06, 'adam_epsilon': 1e-07, 'adam_beta1': 0.8768956788597737, 'adam_beta2': 0.9920981141573957}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.9/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 10/infinite (1 PENDING, 1 RUNNING, 8 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_b6543fec completed. Last result: loss=0.5275164842605591,matthews_correlation=0.37917684067701946\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 59.49ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 39.12ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 106.86ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 110.39ba/s]\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m {'train_runtime': 116.5658, 'train_samples_per_second': 6.692, 'epoch': 2.91}\n",- "\u001b[2m\u001b[36m(pid=914582)\u001b[0m {'eval_loss': 0.5162246823310852, 'eval_matthews_correlation': 0.417156672319181, 'eval_runtime': 1.6762, 'eval_samples_per_second': 622.252, 'epoch': 2.91}\n",- "Trial train_distilbert_0071f998 reported loss=0.5162246823310852,matthews_correlation=0.417156672319181 with parameters={'num_train_epochs': 2.9093911031251687, 'learning_rate': 5.509981405340389e-06, 'adam_epsilon': 7.397757073991268e-08, 'adam_beta1': 0.9550237998045454, 'adam_beta2': 0.9916866638359256}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.0/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 11/infinite (1 PENDING, 1 RUNNING, 9 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_0071f998 completed. Last result: loss=0.5162246823310852,matthews_correlation=0.417156672319181\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 53.62ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 35.94ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 104.02ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 107.63ba/s]\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m {'train_runtime': 64.0869, 'train_samples_per_second': 6.413, 'epoch': 1.53}\n",- "\u001b[2m\u001b[36m(pid=918301)\u001b[0m {'eval_loss': 0.5516289472579956, 'eval_matthews_correlation': 0.06558874629318973, 'eval_runtime': 1.7231, 'eval_samples_per_second': 605.3, 'epoch': 1.53}\n",- "Trial train_distilbert_2f830be6 reported loss=0.5516289472579956,matthews_correlation=0.06558874629318973 with parameters={'num_train_epochs': 1.533382973758465, 'learning_rate': 4.376455192665657e-06, 'adam_epsilon': 7.612697083213253e-08, 'adam_beta1': 0.8869305298731159, 'adam_beta2': 0.9896280392288217}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.2/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 12/infinite (1 PENDING, 1 RUNNING, 10 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_2f830be6 completed. Last result: loss=0.5516289472579956,matthews_correlation=0.06558874629318973\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 59.90ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 39.68ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 84.51ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 86.07ba/s]\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m {'train_runtime': 164.3921, 'train_samples_per_second': 6.667, 'epoch': 4.09}\n",- "\u001b[2m\u001b[36m(pid=920414)\u001b[0m {'eval_loss': 0.523731529712677, 'eval_matthews_correlation': 0.45354879777314566, 'eval_runtime': 1.6345, 'eval_samples_per_second': 638.117, 'epoch': 4.09}\n",- "Trial train_distilbert_7ce03f12 reported loss=0.523731529712677,matthews_correlation=0.45354879777314566 with parameters={'num_train_epochs': 4.087746394379008, 'learning_rate': 7.82557368974717e-06, 'adam_epsilon': 8.701436966404051e-08, 'adam_beta1': 0.9840530092635891, 'adam_beta2': 0.9939557025262034}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.7/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 13/infinite (1 PENDING, 1 RUNNING, 11 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_7ce03f12 completed. Last result: loss=0.523731529712677,matthews_correlation=0.45354879777314566\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 63.59ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 41.23ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 102.78ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 107.25ba/s]\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m {'train_runtime': 135.9594, 'train_samples_per_second': 6.708, 'epoch': 3.4}\n",- "\u001b[2m\u001b[36m(pid=925520)\u001b[0m {'eval_loss': 0.5112878680229187, 'eval_matthews_correlation': 0.4508496945113286, 'eval_runtime': 1.553, 'eval_samples_per_second': 671.602, 'epoch': 3.4}\n",- "Trial train_distilbert_aaab0508 reported loss=0.5112878680229187,matthews_correlation=0.4508496945113286 with parameters={'num_train_epochs': 3.402431917111274, 'learning_rate': 8.91979274640535e-06, 'adam_epsilon': 1e-07, 'adam_beta1': 0.9407072906865396, 'adam_beta2': 0.993946172640627}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 32.3/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 14/infinite (1 PENDING, 1 RUNNING, 12 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_aaab0508 completed. Last result: loss=0.5112878680229187,matthews_correlation=0.4508496945113286\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 57.22ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 38.97ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 90.88ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 89.31ba/s]\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m {'train_runtime': 142.1621, 'train_samples_per_second': 6.591, 'epoch': 3.5}\n",- "\u001b[2m\u001b[36m(pid=929827)\u001b[0m {'eval_loss': 0.5350601673126221, 'eval_matthews_correlation': 0.40085080763525827, 'eval_runtime': 1.7316, 'eval_samples_per_second': 602.346, 'epoch': 3.5}\n",- "Trial train_distilbert_14262454 reported loss=0.5350601673126221,matthews_correlation=0.40085080763525827 with parameters={'num_train_epochs': 3.495397786456084, 'learning_rate': 4.834054640339551e-06, 'adam_epsilon': 4.600931186189709e-08, 'adam_beta1': 0.99, 'adam_beta2': 0.9916961720574915}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.2/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 15/infinite (1 PENDING, 1 RUNNING, 13 TERMINATED)<br><br>"- },- "metadata": {}- },- {+ "name": "stderr","output_type": "stream",- "name": "stdout","text": [- "Trial train_distilbert_14262454 completed. Last result: loss=0.5350601673126221,matthews_correlation=0.40085080763525827\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 53.04ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 37.06ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 106.60ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 90.49ba/s]\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m {'train_runtime': 261.0489, 'train_samples_per_second': 6.83, 'epoch': 6.65}\n",- "\u001b[2m\u001b[36m(pid=934238)\u001b[0m {'eval_loss': 0.609851062297821, 'eval_matthews_correlation': 0.5268023551875569, 'eval_runtime': 1.7076, 'eval_samples_per_second': 610.801, 'epoch': 6.65}\n",- "Trial train_distilbert_6d211fe6 reported loss=0.609851062297821,matthews_correlation=0.5268023551875569 with parameters={'num_train_epochs': 6.649954795358705, 'learning_rate': 1.173331172454689e-05, 'adam_epsilon': 5.4079124274855485e-08, 'adam_beta1': 0.9592773086704482, 'adam_beta2': 0.9945564008561629}.\n"+ "/home/ec2-user/miniconda3/envs/myflaml/lib/python3.8/site-packages/ray/_private/services.py:238: UserWarning: Not all Ray Dashboard dependencies were found. To use the dashboard please install Ray using `pip install ray[default]`. To disable this message, set RAY_DISABLE_IMPORT_WARNING env var to '1'.\n",+ " warnings.warn(warning_message)\n",+ "2021-12-01 23:35:54,348\tWARNING function_runner.py:558 -- Function checkpointing is disabled. This may result in unexpected behavior when using checkpointing features or certain schedulers. To enable, set the train function arguments to be `func(config, checkpoint_dir=None)`.\n"]},{- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.8/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 16/infinite (1 PENDING, 1 RUNNING, 14 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream","name": "stdout",- "text": [- "Trial train_distilbert_6d211fe6 completed. Last result: loss=0.609851062297821,matthews_correlation=0.5268023551875569\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 62.15ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 40.78ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 108.16ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 107.36ba/s]\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m {'train_runtime': 101.88, 'train_samples_per_second': 6.616, 'epoch': 2.51}\n",- "\u001b[2m\u001b[36m(pid=942628)\u001b[0m {'eval_loss': 0.5422758460044861, 'eval_matthews_correlation': 0.32496815807366203, 'eval_runtime': 1.7034, 'eval_samples_per_second': 612.32, 'epoch': 2.51}\n",- "Trial train_distilbert_c980bae4 reported loss=0.5422758460044861,matthews_correlation=0.32496815807366203 with parameters={'num_train_epochs': 2.512749499653892, 'learning_rate': 5.2192940076368766e-06, 'adam_epsilon': 1e-07, 'adam_beta1': 0.99, 'adam_beta2': 0.9933553670097386}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.1/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 17/infinite (1 PENDING, 1 RUNNING, 15 TERMINATED)<br><br>"- },- "metadata": {}- },- {"output_type": "stream",- "name": "stdout","text": [- "Trial train_distilbert_c980bae4 completed. Last result: loss=0.5422758460044861,matthews_correlation=0.32496815807366203\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 57.84ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 40.01ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 102.16ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 81.15ba/s]\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m {'train_runtime': 517.8412, 'train_samples_per_second': 7.087, 'epoch': 13.69}\n",- "\u001b[2m\u001b[36m(pid=945904)\u001b[0m {'eval_loss': 0.9238015413284302, 'eval_matthews_correlation': 0.5494735380761103, 'eval_runtime': 1.588, 'eval_samples_per_second': 656.816, 'epoch': 13.69}\n",- "Trial train_distilbert_6d0d29d6 reported loss=0.9238015413284302,matthews_correlation=0.5494735380761103 with parameters={'num_train_epochs': 13.693961965290004, 'learning_rate': 1.1554924963127978e-05, 'adam_epsilon': 9.975195386335823e-08, 'adam_beta1': 0.9657731407927772, 'adam_beta2': 0.9951819129873288}.\n"+ "Tuning started...\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"]},{- "output_type": "display_data","data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 32.2/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 18/infinite (1 PENDING, 1 RUNNING, 16 TERMINATED)<br><br>"+ "text/html": [+ "== Status ==<br>Memory usage on this node: 4.3/7.7 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4.0/4 CPUs, 4.0/4 GPUs, 0.0/2.34 GiB heap, 0.0/1.17 GiB objects<br>Result logdir: /home/ec2-user/FLAML/notebook/logs/train_distilbert_2021-12-01_23-35-54<br>Number of trials: 1/infinite (1 RUNNING)<br><br>"+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]},- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_6d0d29d6 completed. Last result: loss=0.9238015413284302,matthews_correlation=0.5494735380761103\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 78%|███████▊ | 7/9 [00:00<00:00, 66.59ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 44.15ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 125.62ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 119.07ba/s]\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m {'train_runtime': 129.7296, 'train_samples_per_second': 6.675, 'epoch': 3.23}\n",- "\u001b[2m\u001b[36m(pid=973869)\u001b[0m {'eval_loss': 0.5334658622741699, 'eval_matthews_correlation': 0.4513069078434825, 'eval_runtime': 1.7406, 'eval_samples_per_second': 599.205, 'epoch': 3.23}\n",- "Trial train_distilbert_b16ea82a reported loss=0.5334658622741699,matthews_correlation=0.4513069078434825 with parameters={'num_train_epochs': 3.229299080310228, 'learning_rate': 1.1914452449037918e-05, 'adam_epsilon': 2.9318239583972084e-08, 'adam_beta1': 0.9527814765481193, 'adam_beta2': 0.9939312818847007}.\n"- ]+ "metadata": {},+ "output_type": "display_data"},{- "output_type": "display_data","data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.2/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 19/infinite (1 PENDING, 1 RUNNING, 17 TERMINATED)<br><br>"+ "text/html": [+ "== Status ==<br>Memory usage on this node: 4.5/7.7 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4.0/4 CPUs, 4.0/4 GPUs, 0.0/2.34 GiB heap, 0.0/1.17 GiB objects<br>Result logdir: /home/ec2-user/FLAML/notebook/logs/train_distilbert_2021-12-01_23-35-54<br>Number of trials: 2/infinite (1 PENDING, 1 RUNNING)<br><br>"+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]},- "metadata": {}- },- {- "output_type": "stream",- "name": "stdout",- "text": [- "Trial train_distilbert_b16ea82a completed. Last result: loss=0.5334658622741699,matthews_correlation=0.4513069078434825\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 55.88ba/s]\n",- " 89%|████████▉ | 8/9 [00:00<00:00, 33.97ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 39.36ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 94.15ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 105.63ba/s]\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m {'train_runtime': 604.2581, 'train_samples_per_second': 6.911, 'epoch': 15.58}\n",- "\u001b[2m\u001b[36m(pid=978003)\u001b[0m {'eval_loss': 0.9832845330238342, 'eval_matthews_correlation': 0.5699304939602442, 'eval_runtime': 1.7051, 'eval_samples_per_second': 611.691, 'epoch': 15.58}\n",- "Trial train_distilbert_eddf7cc0 reported loss=0.9832845330238342,matthews_correlation=0.5699304939602442 with parameters={'num_train_epochs': 15.580684188655825, 'learning_rate': 1.2851507818900338e-05, 'adam_epsilon': 8.134982521948352e-08, 'adam_beta1': 0.99, 'adam_beta2': 0.9971094424784387}.\n"- ]+ "metadata": {},+ "output_type": "display_data"},{- "output_type": "display_data","data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.2/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 20/infinite (1 PENDING, 1 RUNNING, 18 TERMINATED)<br><br>"+ "text/html": [+ "== Status ==<br>Memory usage on this node: 4.6/7.7 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4.0/4 CPUs, 4.0/4 GPUs, 0.0/2.34 GiB heap, 0.0/1.17 GiB objects<br>Result logdir: /home/ec2-user/FLAML/notebook/logs/train_distilbert_2021-12-01_23-35-54<br>Number of trials: 2/infinite (1 PENDING, 1 RUNNING)<br><br>"+ ],+ "text/plain": [+ "<IPython.core.display.HTML object>"+ ]},- "metadata": {}+ "metadata": {},+ "output_type": "display_data"},{+ "name": "stderr","output_type": "stream",- "name": "stdout","text": [- "Trial train_distilbert_eddf7cc0 completed. Last result: loss=0.9832845330238342,matthews_correlation=0.5699304939602442\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m Reusing dataset glue (/home/ec2-user/.cache/huggingface/datasets/glue/cola/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4)\n"," 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 53.75ba/s]\n",- " 89%|████████▉ | 8/9 [00:00<00:00, 32.34ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 37.56ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 106.80ba/s]\n",+ " 22%|██▏ | 2/9 [00:00<00:00, 19.41ba/s]\n",+ " 56%|█████▌ | 5/9 [00:00<00:00, 20.98ba/s]\n",+ " 89%|████████▉ | 8/9 [00:00<00:00, 21.75ba/s]\n",+ "100%|██████████| 9/9 [00:00<00:00, 24.49ba/s]\n",+ "100%|██████████| 2/2 [00:00<00:00, 42.79ba/s]\n"," 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 106.92ba/s]\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m {'train_runtime': 464.4178, 'train_samples_per_second': 6.946, 'epoch': 12.04}\n",- "\u001b[2m\u001b[36m(pid=1000417)\u001b[0m {'eval_loss': 0.8574612736701965, 'eval_matthews_correlation': 0.5200220944545176, 'eval_runtime': 1.6294, 'eval_samples_per_second': 640.118, 'epoch': 12.04}\n",- "Trial train_distilbert_43008974 reported loss=0.8574612736701965,matthews_correlation=0.5200220944545176 with parameters={'num_train_epochs': 12.035709859477459, 'learning_rate': 1.0389153769735843e-05, 'adam_epsilon': 1e-07, 'adam_beta1': 0.929089139333885, 'adam_beta2': 0.9932581096367817}.\n"+ "100%|██████████| 2/2 [00:00<00:00, 41.48ba/s]\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 31.4/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 4/4 CPUs, 4/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 21/infinite (1 PENDING, 1 RUNNING, 19 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "stream","name": "stdout",- "text": [- "Trial train_distilbert_43008974 completed. Last result: loss=0.8574612736701965,matthews_correlation=0.5200220944545176\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m Reusing dataset glue (/home/chiw/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",- " 0%| | 0/9 [00:00<?, ?ba/s]\n",- " 67%|██████▋ | 6/9 [00:00<00:00, 57.01ba/s]\n",- "100%|██████████| 9/9 [00:00<00:00, 38.68ba/s]\n",- " 0%| | 0/2 [00:00<?, ?ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 94.45ba/s]\n",- "100%|██████████| 2/2 [00:00<00:00, 106.71ba/s]\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m /home/chiw/.local/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:65: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m warnings.warn('Was asked to gather along dimension 0, but all '\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m {'train_runtime': 464.3935, 'train_samples_per_second': 6.878, 'epoch': 11.92}\n",- "2021-05-07 03:42:29,943\tINFO stopper.py:347 -- Reached timeout of 3600 seconds. Stopping all trials.\n",- "\u001b[2m\u001b[36m(pid=1022436)\u001b[0m {'eval_loss': 0.8282045722007751, 'eval_matthews_correlation': 0.5261643004428046, 'eval_runtime': 1.6945, 'eval_samples_per_second': 615.509, 'epoch': 11.92}\n",- "Trial train_distilbert_b3408a4e reported loss=0.8282045722007751,matthews_correlation=0.5261643004428046 with parameters={'num_train_epochs': 11.916545357570307, 'learning_rate': 1.104181373740261e-05, 'adam_epsilon': 4.674408006864194e-08, 'adam_beta1': 0.99, 'adam_beta2': 0.9938089843451122}.\n"- ]- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 32.0/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 22/infinite (22 TERMINATED)<br><br>"- },- "metadata": {}- },- {- "output_type": "display_data",- "data": {- "text/plain": "<IPython.core.display.HTML object>",- "text/html": "== Status ==<br>Memory usage on this node: 32.0/251.6 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/150.39 GiB heap, 0.0/47.22 GiB objects (0/1.0 accelerator_type:V100)<br>Result logdir: /home/chiw/FLAML/notebook/logs/train_distilbert_2021-05-07_02-35-58<br>Number of trials: 22/infinite (22 TERMINATED)<br><table>\n<thead>\n<tr><th>Trial name </th><th>status </th><th>loc </th><th style=\"text-align: right;\"> adam_beta1</th><th style=\"text-align: right;\"> adam_beta2</th><th style=\"text-align: right;\"> adam_epsilon</th><th style=\"text-align: right;\"> learning_rate</th><th style=\"text-align: right;\"> num_train_epochs</th><th style=\"text-align: right;\"> iter</th><th style=\"text-align: right;\"> total time (s)</th><th style=\"text-align: right;\"> loss</th><th style=\"text-align: right;\"> matthews_correlation</th></tr>\n</thead>\n<tbody>\n<tr><td>train_distilbert_a0c303d0</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.939079</td><td style=\"text-align: right;\"> 0.991865</td><td style=\"text-align: right;\"> 7.96945e-08</td><td style=\"text-align: right;\"> 5.61152e-06</td><td style=\"text-align: right;\"> 1 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 55.6909</td><td style=\"text-align: right;\">0.587986</td><td style=\"text-align: right;\"> 0 </td></tr>\n<tr><td>train_distilbert_a0c303d1</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.811036</td><td style=\"text-align: right;\"> 0.997214</td><td style=\"text-align: right;\"> 2.05111e-09</td><td style=\"text-align: right;\"> 2.05134e-06</td><td style=\"text-align: right;\"> 1.44427</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 71.7663</td><td style=\"text-align: right;\">0.603018</td><td style=\"text-align: right;\"> 0 </td></tr>\n<tr><td>train_distilbert_c39b2ef0</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.909395</td><td style=\"text-align: right;\"> 0.993715</td><td style=\"text-align: right;\"> 1e-07 </td><td style=\"text-align: right;\"> 5.26543e-06</td><td style=\"text-align: right;\"> 1 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 53.7619</td><td style=\"text-align: right;\">0.586518</td><td style=\"text-align: right;\"> 0 </td></tr>\n<tr><td>train_distilbert_f00776e2</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.968763</td><td style=\"text-align: right;\"> 0.990019</td><td style=\"text-align: right;\"> 4.38943e-08</td><td style=\"text-align: right;\"> 5.98035e-06</td><td style=\"text-align: right;\"> 1.02723</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 56.8382</td><td style=\"text-align: right;\">0.581313</td><td style=\"text-align: right;\"> 0 </td></tr>\n<tr><td>train_distilbert_11ab3900</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.962198</td><td style=\"text-align: right;\"> 0.991838</td><td style=\"text-align: right;\"> 7.09296e-08</td><td style=\"text-align: right;\"> 5.06608e-06</td><td style=\"text-align: right;\"> 1 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 54.0231</td><td style=\"text-align: right;\">0.585576</td><td style=\"text-align: right;\"> 0 </td></tr>\n<tr><td>train_distilbert_353025b6</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.91596 </td><td style=\"text-align: right;\"> 0.991892</td><td style=\"text-align: right;\"> 8.95426e-08</td><td style=\"text-align: right;\"> 6.21568e-06</td><td style=\"text-align: right;\"> 2.15443</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 98.3233</td><td style=\"text-align: right;\">0.531632</td><td style=\"text-align: right;\"> 0.388893 </td></tr>\n<tr><td>train_distilbert_5728a1de</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.926933</td><td style=\"text-align: right;\"> 0.993146</td><td style=\"text-align: right;\"> 1e-07 </td><td style=\"text-align: right;\"> 1.00902e-05</td><td style=\"text-align: right;\"> 1 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 55.3726</td><td style=\"text-align: right;\">0.538505</td><td style=\"text-align: right;\"> 0.280558 </td></tr>\n<tr><td>train_distilbert_9394c2e2</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.928106</td><td style=\"text-align: right;\"> 0.990614</td><td style=\"text-align: right;\"> 4.49975e-08</td><td style=\"text-align: right;\"> 3.45674e-06</td><td style=\"text-align: right;\"> 2.72935</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 121.388 </td><td style=\"text-align: right;\">0.539177</td><td style=\"text-align: right;\"> 0.327295 </td></tr>\n<tr><td>train_distilbert_b6543fec</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.876896</td><td style=\"text-align: right;\"> 0.992098</td><td style=\"text-align: right;\"> 1e-07 </td><td style=\"text-align: right;\"> 7.01176e-06</td><td style=\"text-align: right;\"> 1.59538</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 76.0244</td><td style=\"text-align: right;\">0.527516</td><td style=\"text-align: right;\"> 0.379177 </td></tr>\n<tr><td>train_distilbert_0071f998</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.955024</td><td style=\"text-align: right;\"> 0.991687</td><td style=\"text-align: right;\"> 7.39776e-08</td><td style=\"text-align: right;\"> 5.50998e-06</td><td style=\"text-align: right;\"> 2.90939</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 126.871 </td><td style=\"text-align: right;\">0.516225</td><td style=\"text-align: right;\"> 0.417157 </td></tr>\n<tr><td>train_distilbert_2f830be6</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.886931</td><td style=\"text-align: right;\"> 0.989628</td><td style=\"text-align: right;\"> 7.6127e-08 </td><td style=\"text-align: right;\"> 4.37646e-06</td><td style=\"text-align: right;\"> 1.53338</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 73.8934</td><td style=\"text-align: right;\">0.551629</td><td style=\"text-align: right;\"> 0.0655887</td></tr>\n<tr><td>train_distilbert_7ce03f12</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.984053</td><td style=\"text-align: right;\"> 0.993956</td><td style=\"text-align: right;\"> 8.70144e-08</td><td style=\"text-align: right;\"> 7.82557e-06</td><td style=\"text-align: right;\"> 4.08775</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 174.027 </td><td style=\"text-align: right;\">0.523732</td><td style=\"text-align: right;\"> 0.453549 </td></tr>\n<tr><td>train_distilbert_aaab0508</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.940707</td><td style=\"text-align: right;\"> 0.993946</td><td style=\"text-align: right;\"> 1e-07 </td><td style=\"text-align: right;\"> 8.91979e-06</td><td style=\"text-align: right;\"> 3.40243</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 146.249 </td><td style=\"text-align: right;\">0.511288</td><td style=\"text-align: right;\"> 0.45085 </td></tr>\n<tr><td>train_distilbert_14262454</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.99 </td><td style=\"text-align: right;\"> 0.991696</td><td style=\"text-align: right;\"> 4.60093e-08</td><td style=\"text-align: right;\"> 4.83405e-06</td><td style=\"text-align: right;\"> 3.4954 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 152.008 </td><td style=\"text-align: right;\">0.53506 </td><td style=\"text-align: right;\"> 0.400851 </td></tr>\n<tr><td>train_distilbert_6d211fe6</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.959277</td><td style=\"text-align: right;\"> 0.994556</td><td style=\"text-align: right;\"> 5.40791e-08</td><td style=\"text-align: right;\"> 1.17333e-05</td><td style=\"text-align: right;\"> 6.64995</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 271.444 </td><td style=\"text-align: right;\">0.609851</td><td style=\"text-align: right;\"> 0.526802 </td></tr>\n<tr><td>train_distilbert_c980bae4</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.99 </td><td style=\"text-align: right;\"> 0.993355</td><td style=\"text-align: right;\"> 1e-07 </td><td style=\"text-align: right;\"> 5.21929e-06</td><td style=\"text-align: right;\"> 2.51275</td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 111.799 </td><td style=\"text-align: right;\">0.542276</td><td style=\"text-align: right;\"> 0.324968 </td></tr>\n<tr><td>train_distilbert_6d0d29d6</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.965773</td><td style=\"text-align: right;\"> 0.995182</td><td style=\"text-align: right;\"> 9.9752e-08 </td><td style=\"text-align: right;\"> 1.15549e-05</td><td style=\"text-align: right;\"> 13.694 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 527.944 </td><td style=\"text-align: right;\">0.923802</td><td style=\"text-align: right;\"> 0.549474 </td></tr>\n<tr><td>train_distilbert_b16ea82a</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.952781</td><td style=\"text-align: right;\"> 0.993931</td><td style=\"text-align: right;\"> 2.93182e-08</td><td style=\"text-align: right;\"> 1.19145e-05</td><td style=\"text-align: right;\"> 3.2293 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 139.844 </td><td style=\"text-align: right;\">0.533466</td><td style=\"text-align: right;\"> 0.451307 </td></tr>\n<tr><td>train_distilbert_eddf7cc0</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.99 </td><td style=\"text-align: right;\"> 0.997109</td><td style=\"text-align: right;\"> 8.13498e-08</td><td style=\"text-align: right;\"> 1.28515e-05</td><td style=\"text-align: right;\"> 15.5807 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 614.789 </td><td style=\"text-align: right;\">0.983285</td><td style=\"text-align: right;\"> 0.56993 </td></tr>\n<tr><td>train_distilbert_43008974</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.929089</td><td style=\"text-align: right;\"> 0.993258</td><td style=\"text-align: right;\"> 1e-07 </td><td style=\"text-align: right;\"> 1.03892e-05</td><td style=\"text-align: right;\"> 12.0357 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 474.387 </td><td style=\"text-align: right;\">0.857461</td><td style=\"text-align: right;\"> 0.520022 </td></tr>\n<tr><td>train_distilbert_b3408a4e</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.99 </td><td style=\"text-align: right;\"> 0.993809</td><td style=\"text-align: right;\"> 4.67441e-08</td><td style=\"text-align: right;\"> 1.10418e-05</td><td style=\"text-align: right;\"> 11.9165 </td><td style=\"text-align: right;\"> 1</td><td style=\"text-align: right;\"> 474.126 </td><td style=\"text-align: right;\">0.828205</td><td style=\"text-align: right;\"> 0.526164 </td></tr>\n<tr><td>train_distilbert_cfbfb220</td><td>TERMINATED</td><td> </td><td style=\"text-align: right;\"> 0.979454</td><td style=\"text-align: right;\"> 0.9999 </td><td style=\"text-align: right;\"> 1e-07 </td><td style=\"text-align: right;\"> 1.49578e-05</td><td style=\"text-align: right;\"> 20.3715 </td><td style=\"text-align: right;\"> </td><td style=\"text-align: right;\"> </td><td style=\"text-align: right;\"> </td><td style=\"text-align: right;\"> </td></tr>\n</tbody>\n</table><br><br>"- },- "metadata": {}- },- {"output_type": "stream",- "name": "stderr","text": [- "2021-05-07 03:42:30,035\tINFO tune.py:450 -- Total run time: 3992.00 seconds (3991.90 seconds for the tuning loop).\n"+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m To disable this warning, you can either:\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m \t- Avoid using `tokenizers` before the fork if possible\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m \t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m To disable this warning, you can either:\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m \t- Avoid using `tokenizers` before the fork if possible\n",+ "\u001b[2m\u001b[36m(pid=11344)\u001b[0m \t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"]}],@@ -1525,8 +893,7 @@" metric=HP_METRIC,\n"," mode=MODE,\n"," low_cost_partial_config={\"num_train_epochs\": 1}),\n",- " report_intermediate_result=False,\n",- " # uncomment the following if report_intermediate_result = True\n",+ " # uncomment the following if scheduler = 'auto',\n"," # max_resource=max_num_epoch, min_resource=1,\n"," resources_per_trial={\"gpu\": num_gpus, \"cpu\": num_cpus},\n"," local_dir='logs/',\n",@@ -1540,14 +907,17 @@},{"cell_type": "code",- "execution_count": 26,+ "execution_count": null,"metadata": {},"outputs": [{- "output_type": "stream","name": "stdout",+ "output_type": "stream","text": [- "n_trials=22\ntime=3999.769361972809\nBest model eval matthews_correlation: 0.5699\nBest model parameters: {'num_train_epochs': 15.580684188655825, 'learning_rate': 1.2851507818900338e-05, 'adam_epsilon': 8.134982521948352e-08, 'adam_beta1': 0.99, 'adam_beta2': 0.9971094424784387}\n"+ "n_trials=22\n",+ "time=3999.769361972809\n",+ "Best model eval matthews_correlation: 0.5699\n",+ "Best model parameters: {'num_train_epochs': 15.580684188655825, 'learning_rate': 1.2851507818900338e-05, 'adam_epsilon': 8.134982521948352e-08, 'adam_beta1': 0.99, 'adam_beta2': 0.9971094424784387}\n"]}],@@ -1561,6 +931,8 @@]},{+ "cell_type": "markdown",+ "metadata": {},"source": ["## Next Steps\n","\n",@@ -1568,15 +940,17 @@"\n","- Huggingface provides _Callbacks_ which can be used to insert the `flaml.tune.report` call inside the training loop\n","- Make sure to set `do_eval=True` in the `TrainingArguments` provided to `Trainer` and adjust the evaluation frequency accordingly"- ],- "cell_type": "markdown",- "metadata": {}+ ]}],"metadata": {+ "interpreter": {+ "hash": "1cfcceddaeccda27c3cce104660d474924e2ba82887c0e8e481b6ede3743c483"+ },"kernelspec": {- "name": "python385jvsc74a57bd031f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6",- "display_name": "Python 3.8.5 64-bit"+ "display_name": "Python 3.8.5 64-bit",+ "language": "python",+ "name": "python3"},"language_info": {"codemirror_mode": {@@ -1588,7 +962,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.8.5"+ "version": "3.8.12"},"metadata": {"interpreter": {@@ -1598,4 +972,4 @@},"nbformat": 4,"nbformat_minor": 4-}\ No newline at end of file+}diff --git a/notebook/flaml_pytorch_cifar10.ipynb b/notebook/flaml_pytorch_cifar10.ipynbindex ec89f929b..7e096607b 100644--- a/notebook/flaml_pytorch_cifar10.ipynb+++ b/notebook/flaml_pytorch_cifar10.ipynb@@ -286,7 +286,7 @@"metadata": {},"outputs": [],"source": [- "time_budget_s = 600 # time budget in seconds\n",+ "time_budget_s = 3600 # time budget in seconds\n","gpus_per_trial = 0.5 # number of gpus for each trial; 0.5 means two training jobs can share one gpu\n","num_samples = 500 # maximal number of trials\n","np.random.seed(7654321)"@@ -315,7 +315,7 @@" low_cost_partial_config={\"num_epochs\": 1},\n"," max_resource=max_num_epoch,\n"," min_resource=1,\n",- " report_intermediate_result=True, # only set to True when intermediate results are reported by tune.report\n",+ " scheduler=\"asha\", # need to use tune.report to report intermediate results in training_function \n"," resources_per_trial={\"cpu\": 1, \"gpu\": gpus_per_trial},\n"," local_dir='logs/',\n"," num_samples=num_samples,\n",@@ -325,24 +325,9 @@},{"cell_type": "code",- "execution_count": 13,+ "execution_count": null,"metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "#trials=44\n",- "time=1193.913584947586\n",- "Best trial config: {'l1': 8, 'l2': 8, 'lr': 0.0008818671030627281, 'num_epochs': 55.9513429004283, 'batch_size': 3}\n",- "Best trial final validation loss: 1.0694482081472874\n",- "Best trial final validation accuracy: 0.6389\n",- "Files already downloaded and verified\n",- "Files already downloaded and verified\n",- "Best trial test set accuracy: 0.6294\n"- ]- }- ],+ "outputs": [],"source": ["print(f\"#trials={len(result.trials)}\")\n","print(f\"time={time.time()-start_time}\")\n",@@ -390,7 +375,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.7.12"+ "version": "3.8.12"},"metadata": {"interpreter": {diff --git a/test/automl/test_forecast.py b/test/automl/test_forecast.pyindex f582d1119..5befb5307 100644--- a/test/automl/test_forecast.py+++ b/test/automl/test_forecast.py@@ -71,7 +71,7 @@ def test_forecast_automl(budget=5):) = get_output_from_log(filename=settings["log_file_name"], time_budget=budget)for config in config_history:print(config)- print(automl.prune_attr)+ print(automl.resource_attr)print(automl.max_resource)print(automl.min_resource)@@ -210,7 +210,7 @@ def test_multivariate_forecast_num(budget=5):) = get_output_from_log(filename=settings["log_file_name"], time_budget=budget)for config in config_history:print(config)- print(automl.prune_attr)+ print(automl.resource_attr)print(automl.max_resource)print(automl.min_resource)@@ -341,7 +341,7 @@ def test_multivariate_forecast_cat(budget=5):) = get_output_from_log(filename=settings["log_file_name"], time_budget=budget)for config in config_history:print(config)- print(automl.prune_attr)+ print(automl.resource_attr)print(automl.max_resource)print(automl.min_resource)diff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.pyindex 515d569b1..33c9c15ce 100644--- a/test/automl/test_notebook_example.py+++ b/test/automl/test_notebook_example.py@@ -64,7 +64,7 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):) = get_output_from_log(filename=settings["log_file_name"], time_budget=6)for config in config_history:print(config)- print(automl.prune_attr)+ print(automl.resource_attr)print(automl.max_resource)print(automl.min_resource)diff --git a/test/automl/test_python_log.py b/test/automl/test_python_log.pyindex c367600c1..3681fcd45 100644--- a/test/automl/test_python_log.py+++ b/test/automl/test_python_log.py@@ -80,7 +80,7 @@ class TestLogging(unittest.TestCase):low_cost_partial_config=low_cost_partial_config,points_to_evaluate=automl.points_to_evaluate,cat_hp_cost=automl.cat_hp_cost,- prune_attr=automl.prune_attr,+ resource_attr=automl.resource_attr,min_resource=automl.min_resource,max_resource=automl.max_resource,config_constraints=[diff --git a/test/automl/test_xgboost2d.py b/test/automl/test_xgboost2d.pyindex 2c17850a0..992420cd4 100644--- a/test/automl/test_xgboost2d.py+++ b/test/automl/test_xgboost2d.py@@ -71,7 +71,7 @@ def test_simple(method=None):low_cost_partial_config=automl.low_cost_partial_config,points_to_evaluate=automl.points_to_evaluate,cat_hp_cost=automl.cat_hp_cost,- prune_attr=automl.prune_attr,+ resource_attr=automl.resource_attr,min_resource=automl.min_resource,max_resource=automl.max_resource,time_budget_s=automl._state.time_budget,diff --git a/test/tune/test_pytorch_cifar10.py b/test/tune/test_pytorch_cifar10.pyindex 3087f4548..2151bf281 100644--- a/test/tune/test_pytorch_cifar10.py+++ b/test/tune/test_pytorch_cifar10.py@@ -239,7 +239,7 @@ def cifar10_main(low_cost_partial_config={"num_epochs": 1},max_resource=max_num_epochs,min_resource=1,- report_intermediate_result=True,+ scheduler="asha",resources_per_trial={"cpu": 1, "gpu": gpus_per_trial},local_dir="logs/",num_samples=num_samples,diff --git a/test/tune/test_scheduler.py b/test/tune/test_scheduler.pynew file mode 100644index 000000000..906d03619--- /dev/null+++ b/test/tune/test_scheduler.py@@ -0,0 +1,157 @@+"""Require: pip install flaml[test,ray]+"""+from logging import raiseExceptions+from flaml.scheduler.trial_scheduler import TrialScheduler+import numpy as np+from flaml import tune+import time+++def rand_vector_unit_sphere(dim):+ """this function allows you to generate+ points that uniformly distribute on+ the (dim-1)-sphere.+ """+ vec = np.random.normal(0, 1, dim)+ mag = np.linalg.norm(vec)+ return vec / mag+++def simple_obj(config, resource=10000):+ config_value_vector = np.array([config["x"], config["y"], config["z"]])+ score_sequence = []+ for i in range(resource):+ a = rand_vector_unit_sphere(3)+ a[2] = abs(a[2])+ point_projection = np.dot(config_value_vector, a)+ score_sequence.append(point_projection)+ score_avg = np.mean(np.array(score_sequence))+ score_std = np.std(np.array(score_sequence))+ score_lb = score_avg - 1.96 * score_std / np.sqrt(resource)+ tune.report(samplesize=resource, sphere_projection=score_lb)+++def obj_w_intermediate_report(resource, config):+ config_value_vector = np.array([config["x"], config["y"], config["z"]])+ score_sequence = []+ for i in range(resource):+ a = rand_vector_unit_sphere(3)+ a[2] = abs(a[2])+ point_projection = np.dot(config_value_vector, a)+ score_sequence.append(point_projection)+ if (i + 1) % 100 == 0:+ score_avg = np.mean(np.array(score_sequence))+ score_std = np.std(np.array(score_sequence))+ score_lb = score_avg - 1.96 * score_std / np.sqrt(i + 1)+ tune.report(samplesize=i + 1, sphere_projection=score_lb)+++def obj_w_suggested_resource(resource_attr, config):+ resource = config[resource_attr]+ simple_obj(config, resource)+++def test_scheduler(scheduler=None):+ from functools import partial++ resource_attr = "samplesize"+ max_resource = 10000++ # specify the objective functions+ if scheduler is None:+ evaluation_obj = simple_obj+ elif scheduler == "flaml":+ evaluation_obj = partial(obj_w_suggested_resource, resource_attr)+ elif scheduler == "asha" or isinstance(scheduler, TrialScheduler):+ evaluation_obj = partial(obj_w_intermediate_report, max_resource)+ else:+ try:+ from ray.tune.schedulers import TrialScheduler as RayTuneTrialScheduler+ except ImportError:+ print(+ "skip this condition, which may require TrialScheduler from ray tune, \+ as ray tune cannot be imported."+ )+ return+ if isinstance(scheduler, RayTuneTrialScheduler):+ evaluation_obj = partial(obj_w_intermediate_report, max_resource)+ else:+ raise ValueError++ analysis = tune.run(+ evaluation_obj,+ config={+ "x": tune.uniform(5, 20),+ "y": tune.uniform(0, 10),+ "z": tune.uniform(0, 10),+ },+ metric="sphere_projection",+ mode="max",+ verbose=1,+ resource_attr=resource_attr,+ scheduler=scheduler,+ max_resource=max_resource,+ min_resource=100,+ reduction_factor=2,+ time_budget_s=1,+ num_samples=500,+ )++ print("Best hyperparameters found were: ", analysis.best_config)+ # print(analysis.get_best_trial)+ return analysis.best_config+++def test_no_scheduler():+ best_config = test_scheduler()+ print("No scheduler, test error:", abs(10 / 2 - best_config["z"] / 2))+++def test_asha_scheduler():+ try:+ from ray.tune.schedulers import ASHAScheduler+ except ImportError:+ print("skip the test as ray tune cannot be imported.")+ return+ best_config = test_scheduler(scheduler="asha")+ print("Auto ASHA scheduler, test error:", abs(10 / 2 - best_config["z"] / 2))+++def test_custom_scheduler():+ try:+ from ray.tune.schedulers import HyperBandScheduler+ except ImportError:+ print("skip the test as ray tune cannot be imported.")+ return+ my_scheduler = HyperBandScheduler(+ time_attr="samplesize", max_t=1000, reduction_factor=2+ )+ best_config = test_scheduler(scheduler=my_scheduler)+ print("Custom ASHA scheduler, test error:", abs(10 / 2 - best_config["z"] / 2))+++def test_custom_scheduler_default_time_attr():+ try:+ from ray.tune.schedulers import ASHAScheduler+ except ImportError:+ print("skip the test as ray tune cannot be imported.")+ return+ my_scheduler = ASHAScheduler(max_t=10)+ best_config = test_scheduler(scheduler=my_scheduler)+ print(+ "Custom ASHA scheduler (with ASHA default time attr), test error:",+ abs(10 / 2 - best_config["z"] / 2),+ )+++def test_flaml_scheduler():+ best_config = test_scheduler(scheduler="flaml")+ print("FLAML scheduler, test error", abs(10 / 2 - best_config["z"] / 2))+++if __name__ == "__main__":+ test_no_scheduler()+ test_asha_scheduler()+ test_custom_scheduler()+ test_custom_scheduler_default_time_attr()+ test_flaml_scheduler()diff --git a/test/tune/test_tune.py b/test/tune/test_tune.pyindex d4bfbd24b..ab7232a08 100644--- a/test/tune/test_tune.py+++ b/test/tune/test_tune.py@@ -83,7 +83,7 @@ def _test_xgboost(method="BlendSearch"):mode="min",max_resource=max_iter,min_resource=1,- report_intermediate_result=True,+ scheduler="asha",# You can add "gpu": 0.1 to allocate GPUsresources_per_trial={"cpu": 1},local_dir="logs/",