A programming framework for agentic AI
fixing trainable and update function, completing NOTE (#566)
Xueqing Liue0e317bfb1be
diff --git a/flaml/automl.py b/flaml/automl.pyindex eb4e19619..564470795 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -237,7 +237,7 @@ class SearchState:if self.base_eci is None:self.base_eci = time_used- if (obj is not None) and (self.best_loss is None or obj < self.best_loss):+ if (obj is not None) and (obj < self.best_loss):self.best_loss_old = self.best_loss if self.best_loss < np.inf else 2 * objself.best_loss = objself.best_result = result@@ -286,7 +286,7 @@ class AutoMLState:sampled_y_train = self.y_train[:sample_size]weight = self.fit_kwargs.get("sample_weight"- ) # NOTE: _prepare_sample_train_data is before+ ) # NOTE: _prepare_sample_train_data is before kwargs is updated to fit_kwargs_by_estimatorif weight is not None:sampled_weight = weight[:sample_size]if self.groups is not None:@@ -296,7 +296,7 @@ class AutoMLState:sampled_y_train = self.y_train_allif ("sample_weight" in self.fit_kwargs- ): # NOTE: _prepare_sample_train_data is before+ ): # NOTE: _prepare_sample_train_data is before kwargs is updated to fit_kwargs_by_estimatorsampled_weight = self.sample_weight_allif self.groups is not None:groups = self.groups_all@@ -311,7 +311,7 @@ class AutoMLState:this_estimator_kwargs = state.fit_kwargs_by_estimator.get(estimator- ).copy() # NOTE: _compute_with_config_base is after+ ).copy() # NOTE: _compute_with_config_base is after kwargs is updated to fit_kwargs_by_estimator(sampled_X_train,sampled_y_train,@@ -398,7 +398,7 @@ class AutoMLState:this_estimator_kwargs = self.fit_kwargs_by_estimator.get(estimator- ).copy() # NOTE: _train_with_config is after+ ).copy() # NOTE: _train_with_config is after kwargs is updated to fit_kwargs_by_estimator(sampled_X_train,sampled_y_train,@@ -408,14 +408,14 @@ class AutoMLState:if sampled_weight is not None:weight = this_estimator_kwargs["sample_weight"- ] # NOTE: _train_with_config is after+ ] # NOTE: _train_with_config is after kwargs is updated to fit_kwargs_by_estimatorthis_estimator_kwargs["sample_weight"- ] = sampled_weight # NOTE: _train_with_config is after+ ] = sampled_weight # NOTE: _train_with_config is after kwargs is updated to fit_kwargs_by_estimatorif groups is not None:this_estimator_kwargs["groups"- ] = groups # NOTE: _train_with_config is after+ ] = groups # NOTE: _train_with_config is after kwargs is updated to fit_kwargs_by_estimatorbudget = (None@@ -432,14 +432,14 @@ class AutoMLState:n_jobs=self.n_jobs,estimator_class=self.learner_classes.get(estimator),budget=budget,- fit_kwargs=this_estimator_kwargs, # NOTE: _train_with_config is after+ fit_kwargs=this_estimator_kwargs, # NOTE: _train_with_config is after kwargs is updated to fit_kwargs_by_estimatoreval_metric=self.metric if hasattr(self, "metric") else "train_time",)if sampled_weight is not None:this_estimator_kwargs["sample_weight"- ] = weight # NOTE: _train_with_config is after+ ] = weight # NOTE: _train_with_config is after kwargs is updated to fit_kwargs_by_estimatorreturn estimator, train_time@@ -626,10 +626,12 @@ class AutoML(BaseEstimator):augment rare classes.min_sample_size: int, default=MIN_SAMPLE_TRAIN | the minimal samplesize when sample=True.- use_ray: boolean, default=False | Whether to use ray to run the training+ use_ray: boolean or dict.+ If boolean: default=False | Whether to use ray to run the trainingin separate processes. This can be used to prevent OOM for large- datasets, but will incur more overhead in time. Only use it if- you run into OOM failures.+ datasets, but will incur more overhead in time.+ If dict: the dict contains the keywords arguments to be passed to+ [ray.tune.run](https://docs.ray.io/en/latest/tune/api_docs/execution.html).metric_constraints: list, default=[] | The list of metric constraints.Each element in this list is a 3-tuple, which shall be expressedin the following format: the first element of the 3-tuple is the name of the@@ -1103,7 +1105,7 @@ class AutoML(BaseEstimator):self._sample_weight_full = self._state.fit_kwargs.get("sample_weight"- ) # NOTE: _validate_data is before,+ ) # NOTE: _validate_data is before kwargs is updated to fit_kwargs_by_estimatorif X_val is not None and y_val is not None:assert (isinstance(X_val, np.ndarray)@@ -1164,7 +1166,7 @@ class AutoML(BaseEstimator):self._state.task in CLASSIFICATIONand self._auto_augmentand self._state.fit_kwargs.get("sample_weight")- is None # NOTE: _prepare_data is before+ is None # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorand self._split_type in ["stratified", "uniform"]and self._state.task != TOKENCLASSIFICATION):@@ -1208,7 +1210,9 @@ class AutoML(BaseEstimator):)self._state.fit_kwargs["sample_weight"- ] = self._state.sample_weight_all # NOTE: _prepare_data is before+ ] = (+ self._state.sample_weight_all+ ) # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorelse:X_train_all, y_train_all = shuffle(X_train_all, y_train_all, random_state=RANDOM_SEED@@ -1227,7 +1231,7 @@ class AutoML(BaseEstimator):num_samples = X_train_all.shape[0]period = self._state.fit_kwargs["period"- ] # NOTE: _prepare_data is before+ ] # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorassert (period < num_samples), f"period={period}>#examples={num_samples}"@@ -1239,7 +1243,7 @@ class AutoML(BaseEstimator):else:if ("sample_weight" in self._state.fit_kwargs- ): # NOTE: _prepare_data is before+ ): # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator(X_train,X_val,@@ -1247,14 +1251,14 @@ class AutoML(BaseEstimator):y_val,self._state.fit_kwargs["sample_weight"- ], # NOTE: _prepare_data is before+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorself._state.weight_val,) = train_test_split(X_train_all,y_train_all,self._state.fit_kwargs["sample_weight"- ], # NOTE: _prepare_data is before+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatortest_size=split_ratio,shuffle=False,)@@ -1297,7 +1301,7 @@ class AutoML(BaseEstimator):stratify = y_rest if self._split_type == "stratified" else Noneif ("sample_weight" in self._state.fit_kwargs- ): # NOTE: _prepare_data is before+ ): # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator(X_train,X_val,@@ -1310,17 +1314,17 @@ class AutoML(BaseEstimator):y_rest,self._state.fit_kwargs["sample_weight"][rest- ], # NOTE: _prepare_data is before+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatortest_size=split_ratio,random_state=RANDOM_SEED,)weight1 = self._state.fit_kwargs["sample_weight"][first- ] # NOTE: _prepare_data is before+ ] # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorself._state.weight_val = concat(weight1, weight_val)self._state.fit_kwargs["sample_weight"- ] = concat( # NOTE: _prepare_data is before+ ] = concat( # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorweight1, weight_train)else:@@ -1346,7 +1350,7 @@ class AutoML(BaseEstimator):elif self._state.task in REGRESSION:if ("sample_weight" in self._state.fit_kwargs- ): # NOTE: _prepare_data is before+ ): # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator(X_train,X_val,@@ -1354,14 +1358,14 @@ class AutoML(BaseEstimator):y_val,self._state.fit_kwargs["sample_weight"- ], # NOTE: _prepare_data is before+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorself._state.weight_val,) = train_test_split(X_train_all,y_train_all,self._state.fit_kwargs["sample_weight"- ], # NOTE: _prepare_data is before+ ], # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatortest_size=split_ratio,random_state=RANDOM_SEED,)@@ -1409,7 +1413,7 @@ class AutoML(BaseEstimator):if self._state.task in TS_FORECAST:period = self._state.fit_kwargs["period"- ] # NOTE: _prepare_data is before+ ] # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimatorif period * (n_splits + 1) > y_train_all.size:n_splits = int(y_train_all.size / period - 1)assert n_splits >= 2, (@@ -1716,7 +1720,7 @@ class AutoML(BaseEstimator):assert isinstance(self._state.fit_kwargs.get("period"),- int, # NOTE: _decide_split_type is before+ int, # NOTE: _decide_split_type is before kwargs is updated to fit_kwargs_by_estimator), f"missing a required integer 'period' for '{TS_FORECAST}' task."elif self._state.task == "rank":assert (@@ -1897,32 +1901,14 @@ class AutoML(BaseEstimator):@propertydef trainable(self) -> Callable[[dict], Optional[float]]:"""Training function.-Returns:A function that evaluates each config and returns the loss."""self._state.time_from_start = 0- for estimator in self.estimator_list:- search_state = self._search_states[estimator]- if not hasattr(search_state, "training_function"):- if self._use_ray is not False:- from ray.tune import with_parameters-- search_state.training_function = with_parameters(- AutoMLState._compute_with_config_base,- state=self._state,- estimator=estimator,- )- else:- search_state.training_function = partial(- AutoMLState._compute_with_config_base,- state=self._state,- estimator=estimator,- )states = self._search_statesmem_res = self._mem_thres- def train(config: dict):+ def train(config: dict, state):sample_size = config.get("FLAML_sample_size")config = config.get("ml", config).copy()@@ -1932,18 +1918,33 @@ class AutoML(BaseEstimator):# check memory constraints before trainingif states[estimator].learner_class.size(config) <= mem_res:del config["learner"]- result = states[estimator].training_function(config)- return result+ result = AutoMLState._compute_with_config_base(+ config, state=state, estimator=estimator+ )else:- return {+ # If search algorithm is not in flaml, it does not handle the config constraint, should also tune.report before return+ result = {"pred_time": 0,"wall_clock_time": None,"metric_for_logging": np.inf,"val_loss": np.inf,"trained_estimator": None,}+ tune.report(**result)+ return result++ if self._use_ray is not False:+ from ray.tune import with_parameters- return train+ return with_parameters(+ train,+ state=self._state,+ )+ else:+ return partial(+ train,+ state=self._state,+ )@propertydef metric_constraints(self) -> list:@@ -2180,10 +2181,12 @@ class AutoML(BaseEstimator):augment rare classes.min_sample_size: int, default=MIN_SAMPLE_TRAIN | the minimal samplesize when sample=True.- use_ray: boolean or dict+ use_ray: boolean or dict.If boolean: default=False | Whether to use ray to run the trainingin separate processes. This can be used to prevent OOM for largedatasets, but will incur more overhead in time.+ If dict: the dict contains the keywords arguments to be passed to+ [ray.tune.run](https://docs.ray.io/en/latest/tune/api_docs/execution.html).metric_constraints: list, default=[] | The list of metric constraints.Each element in this list is a 3-tuple, which shall be expressedin the following format: the first element of the 3-tuple is the name of the@@ -2565,7 +2568,7 @@ class AutoML(BaseEstimator):this_estimator_kwargs = this_estimator_kwargs.copy()this_estimator_kwargs.update(self._state.fit_kwargs- ) # update the shallow copy+ ) # update the shallow copy of fit_kwargs to fit_kwargs_by_estimatorself._state.fit_kwargs_by_estimator[estimator_name] = this_estimator_kwargs # set self._state.fit_kwargs_by_estimator[estimator_name] to the update, so only self._state.fit_kwargs_by_estimator will be updated@@ -2579,7 +2582,9 @@ class AutoML(BaseEstimator):data_size=self._state.data_size,task=self._state.task,starting_point=starting_points.get(estimator_name),- period=self._state.fit_kwargs.get("period"), # NOTE: this is after+ period=self._state.fit_kwargs.get(+ "period"+ ), # NOTE: this is after kwargs is updated to fit_kwargs_by_estimatorcustom_hp=custom_hp and custom_hp.get(estimator_name),max_iter=max_iter,)@@ -2643,7 +2648,7 @@ class AutoML(BaseEstimator):self._sample_weight_full,self._state.fit_kwargs_by_estimator,self._state.fit_kwargs,- ) # NOTE: this is after+ ) # NOTE: this is after kwargs is updated to fit_kwargs_by_estimatordel self._state.groups, self._state.groups_all, self._state.groups_vallogger.setLevel(old_level)@@ -3257,7 +3262,7 @@ class AutoML(BaseEstimator):stacker.fit(self._X_train_all,self._y_train_all,- **sample_weight_dict, # NOTE: _search is after+ **sample_weight_dict, # NOTE: _search is after kwargs is updated to fit_kwargs_by_estimator)logger.info(f"ensemble: {stacker}")self._trained_estimator = stacker@@ -3276,7 +3281,7 @@ class AutoML(BaseEstimator):stacker.fit(self._X_train_all,self._y_train_all,- **sample_weight_dict, # NOTE: _search is after+ **sample_weight_dict, # NOTE: _search is after kwargs is updated to fit_kwargs_by_estimator)logger.info(f"ensemble: {stacker}")self._trained_estimator = stackerdiff --git a/flaml/ml.py b/flaml/ml.pyindex 092a02565..81301db88 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -420,7 +420,6 @@ def get_val_loss(# fit_kwargs['groups_val'] = groups_val# fit_kwargs['X_val'] = X_val# fit_kwargs['y_val'] = y_val-estimator.fit(X_train, y_train, budget, **fit_kwargs)val_loss, metric_for_logging, pred_time, _ = _eval_estimator(config,diff --git a/test/automl/test_constraints.py b/test/automl/test_constraints.pyindex 53ce52127..a47be3078 100644--- a/test/automl/test_constraints.py+++ b/test/automl/test_constraints.py@@ -21,7 +21,7 @@ def test_metric_constraints():"log_type": "all","retrain_full": "budget","keep_search_state": True,- "time_budget": 1,+ "time_budget": 2,"pred_time_limit": 5.1e-05,}from sklearn.externals._arff import ArffException