A programming framework for agentic AI
Add mlflow_logging param (#1015)
garar31864d2d777b
diff --git a/flaml/automl/automl.py b/flaml/automl/automl.pyindex 5dede562b..e9861ae89 100644--- a/flaml/automl/automl.py+++ b/flaml/automl/automl.py@@ -341,6 +341,9 @@ class AutoML(BaseEstimator):}}```+ mlflow_logging: boolean, default=True | Whether to log the training results to mlflow.+ This requires mlflow to be installed and to have an active mlflow run.+ FLAML will create nested runs."""self._track_iter = 0@@ -390,6 +393,7 @@ class AutoML(BaseEstimator):settings["fit_kwargs_by_estimator"] = settings.get("fit_kwargs_by_estimator", {})settings["custom_hp"] = settings.get("custom_hp", {})settings["skip_transform"] = settings.get("skip_transform", False)+ settings["mlflow_logging"] = settings.get("mlflow_logging", True)self._estimator_type = "classifier" if settings["task"] in CLASSIFICATION else "regressor"@@ -1213,6 +1217,7 @@ class AutoML(BaseEstimator):custom_hp=None,cv_score_agg_func=None,skip_transform=None,+ mlflow_logging=None,fit_kwargs_by_estimator=None,**fit_kwargs,):@@ -1474,6 +1479,11 @@ class AutoML(BaseEstimator):```skip_transform: boolean, default=False | Whether to pre-process data prior to modeling.+ mlflow_logging: boolean, default=None | Whether to log the training results to mlflow.+ Default value is None, which means the logging decision is made based on+ AutoML.__init__'s mlflow_logging argument.+ This requires mlflow to be installed and to have an active mlflow run.+ FLAML will create nested runs.fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name.For TransformersEstimator, available fit_kwargs can be found from[TrainingArgumentsForAuto](nlp/huggingface/training_args).@@ -1659,6 +1669,7 @@ class AutoML(BaseEstimator):self._state.fit_kwargs = fit_kwargscustom_hp = custom_hp or self._settings.get("custom_hp")self._skip_transform = self._settings.get("skip_transform") if skip_transform is None else skip_transform+ self._mlflow_logging = self._settings.get("mlflow_logging") if mlflow_logging is None else mlflow_loggingfit_kwargs_by_estimator = fit_kwargs_by_estimator or self._settings.get("fit_kwargs_by_estimator")self._state.fit_kwargs_by_estimator = fit_kwargs_by_estimator.copy() # shallow copy of fit_kwargs_by_estimatorself._state.weight_val = sample_weight_val@@ -2139,7 +2150,7 @@ class AutoML(BaseEstimator):estimator,search_state.sample_size,)- if mlflow is not None and mlflow.active_run():+ if self._mlflow_logging and mlflow is not None and mlflow.active_run():with mlflow.start_run(nested=True):mlflow.log_metric("iter_counter", self._track_iter)if (search_state.metric_for_logging is not None) and (diff --git a/test/automl/test_mlflow.py b/test/automl/test_mlflow.pynew file mode 100644index 000000000..607ccf696--- /dev/null+++ b/test/automl/test_mlflow.py@@ -0,0 +1,64 @@+import pytest+from pandas import DataFrame+from sklearn.datasets import load_iris+import mlflow+import mlflow.entities+from flaml import AutoML+++class TestMLFlowLoggingParam:+ def test_should_start_new_run_by_default(self, automl_settings):+ with mlflow.start_run():+ parent = mlflow.last_active_run()+ automl = AutoML()+ X_train, y_train = load_iris(return_X_y=True)+ automl.fit(X_train=X_train, y_train=y_train, **automl_settings)++ children = self._get_child_runs(parent)+ assert len(children) >= 1, "Expected at least 1 child run, got {}".format(len(children))++ def test_should_not_start_new_run_when_mlflow_logging_set_to_false_in_init(self, automl_settings):+ with mlflow.start_run():+ parent = mlflow.last_active_run()+ automl = AutoML(mlflow_logging=False)+ X_train, y_train = load_iris(return_X_y=True)+ automl.fit(X_train=X_train, y_train=y_train, **automl_settings)++ children = self._get_child_runs(parent)+ assert len(children) == 0, "Expected 0 child runs, got {}".format(len(children))++ def test_should_not_start_new_run_when_mlflow_logging_set_to_false_in_fit(self, automl_settings):+ with mlflow.start_run():+ parent = mlflow.last_active_run()+ automl = AutoML()+ X_train, y_train = load_iris(return_X_y=True)+ automl.fit(X_train=X_train, y_train=y_train, mlflow_logging=False, **automl_settings)++ children = self._get_child_runs(parent)+ assert len(children) == 0, "Expected 0 child runs, got {}".format(len(children))++ def test_should_start_new_run_when_mlflow_logging_set_to_true_in_fit(self, automl_settings):+ with mlflow.start_run():+ parent = mlflow.last_active_run()+ automl = AutoML(mlflow_logging=False)+ X_train, y_train = load_iris(return_X_y=True)+ automl.fit(X_train=X_train, y_train=y_train, mlflow_logging=True, **automl_settings)++ children = self._get_child_runs(parent)+ assert len(children) >= 1, "Expected at least 1 child run, got {}".format(len(children))++ @staticmethod+ def _get_child_runs(parent_run: mlflow.entities.Run) -> DataFrame:+ experiment_id = parent_run.info.experiment_id+ return mlflow.search_runs(+ [experiment_id], filter_string="tags.mlflow.parentRunId = '{}'".format(parent_run.info.run_id)+ )++ @pytest.fixture(scope="class")+ def automl_settings(self):+ return {+ "time_budget": 2, # in seconds+ "metric": "accuracy",+ "task": "classification",+ "log_file_name": "iris.log",+ }diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.mdindex 453cdd7ca..28b5ce255 100644--- a/website/docs/Use-Cases/Task-Oriented-AutoML.md+++ b/website/docs/Use-Cases/Task-Oriented-AutoML.md@@ -477,6 +477,18 @@ with mlflow.start_run():automl.fit(X_train=X_train, y_train=y_train, **settings)```+To disable mlflow logging pre-configured in FLAML, set `mlflow_logging=False`:+```python+automl = AutoML(mlflow_logging=False)+```+or+```python+automl.fit(X_train=X_train, y_train=y_train, mlflow_logging=False, **settings)+```++Setting `mlflow_logging=False` in the constructor will disable mlflow logging for all the `fit()` calls.+Setting `mlflow_logging=False` in `fit()` will disable mlflow logging for that `fit()` call only.+### Extra fit argumentsExtra fit arguments that are needed by the estimators can be passed to `AutoML.fit()`. For example, if there is a weight associated with each training example, they can be passed via `sample_weight`. For another example, `period` can be passed for time series forecaster. For any extra keywork argument passed to `AutoML.fit()` which has not been explicitly listed in the function signature, it will be passed to the underlying estimators' `fit()` as is. For another example, you can set the number of gpus used by each trial with the `gpu_per_trial` argument, which is only used by TransformersEstimator and XGBoostSklearnEstimator.