Skip to content

A programming framework for agentic AI

Add mlflow_logging param (#1015)

garar31864d2d777b
diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py
index 5dede562b..e9861ae89 100644
--- a/flaml/automl/automl.py
+++ b/flaml/automl/automl.py
@@ -341,6 +341,9 @@ class AutoML(BaseEstimator):
}
}
```
+ mlflow_logging: boolean, default=True | Whether to log the training results to mlflow.
+ This requires mlflow to be installed and to have an active mlflow run.
+ FLAML will create nested runs.
"""
self._track_iter = 0
@@ -390,6 +393,7 @@ class AutoML(BaseEstimator):
settings["fit_kwargs_by_estimator"] = settings.get("fit_kwargs_by_estimator", {})
settings["custom_hp"] = settings.get("custom_hp", {})
settings["skip_transform"] = settings.get("skip_transform", False)
+ settings["mlflow_logging"] = settings.get("mlflow_logging", True)
self._estimator_type = "classifier" if settings["task"] in CLASSIFICATION else "regressor"
@@ -1213,6 +1217,7 @@ class AutoML(BaseEstimator):
custom_hp=None,
cv_score_agg_func=None,
skip_transform=None,
+ mlflow_logging=None,
fit_kwargs_by_estimator=None,
**fit_kwargs,
):
@@ -1474,6 +1479,11 @@ class AutoML(BaseEstimator):
```
skip_transform: boolean, default=False | Whether to pre-process data prior to modeling.
+ mlflow_logging: boolean, default=None | Whether to log the training results to mlflow.
+ Default value is None, which means the logging decision is made based on
+ AutoML.__init__'s mlflow_logging argument.
+ This requires mlflow to be installed and to have an active mlflow run.
+ FLAML will create nested runs.
fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name.
For TransformersEstimator, available fit_kwargs can be found from
[TrainingArgumentsForAuto](nlp/huggingface/training_args).
@@ -1659,6 +1669,7 @@ class AutoML(BaseEstimator):
self._state.fit_kwargs = fit_kwargs
custom_hp = custom_hp or self._settings.get("custom_hp")
self._skip_transform = self._settings.get("skip_transform") if skip_transform is None else skip_transform
+ self._mlflow_logging = self._settings.get("mlflow_logging") if mlflow_logging is None else mlflow_logging
fit_kwargs_by_estimator = fit_kwargs_by_estimator or self._settings.get("fit_kwargs_by_estimator")
self._state.fit_kwargs_by_estimator = fit_kwargs_by_estimator.copy() # shallow copy of fit_kwargs_by_estimator
self._state.weight_val = sample_weight_val
@@ -2139,7 +2150,7 @@ class AutoML(BaseEstimator):
estimator,
search_state.sample_size,
)
- if mlflow is not None and mlflow.active_run():
+ if self._mlflow_logging and mlflow is not None and mlflow.active_run():
with mlflow.start_run(nested=True):
mlflow.log_metric("iter_counter", self._track_iter)
if (search_state.metric_for_logging is not None) and (
diff --git a/test/automl/test_mlflow.py b/test/automl/test_mlflow.py
new file mode 100644
index 000000000..607ccf696
--- /dev/null
+++ b/test/automl/test_mlflow.py
@@ -0,0 +1,64 @@
+import pytest
+from pandas import DataFrame
+from sklearn.datasets import load_iris
+import mlflow
+import mlflow.entities
+from flaml import AutoML
+
+
+class TestMLFlowLoggingParam:
+ def test_should_start_new_run_by_default(self, automl_settings):
+ with mlflow.start_run():
+ parent = mlflow.last_active_run()
+ automl = AutoML()
+ X_train, y_train = load_iris(return_X_y=True)
+ automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
+
+ children = self._get_child_runs(parent)
+ assert len(children) >= 1, "Expected at least 1 child run, got {}".format(len(children))
+
+ def test_should_not_start_new_run_when_mlflow_logging_set_to_false_in_init(self, automl_settings):
+ with mlflow.start_run():
+ parent = mlflow.last_active_run()
+ automl = AutoML(mlflow_logging=False)
+ X_train, y_train = load_iris(return_X_y=True)
+ automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
+
+ children = self._get_child_runs(parent)
+ assert len(children) == 0, "Expected 0 child runs, got {}".format(len(children))
+
+ def test_should_not_start_new_run_when_mlflow_logging_set_to_false_in_fit(self, automl_settings):
+ with mlflow.start_run():
+ parent = mlflow.last_active_run()
+ automl = AutoML()
+ X_train, y_train = load_iris(return_X_y=True)
+ automl.fit(X_train=X_train, y_train=y_train, mlflow_logging=False, **automl_settings)
+
+ children = self._get_child_runs(parent)
+ assert len(children) == 0, "Expected 0 child runs, got {}".format(len(children))
+
+ def test_should_start_new_run_when_mlflow_logging_set_to_true_in_fit(self, automl_settings):
+ with mlflow.start_run():
+ parent = mlflow.last_active_run()
+ automl = AutoML(mlflow_logging=False)
+ X_train, y_train = load_iris(return_X_y=True)
+ automl.fit(X_train=X_train, y_train=y_train, mlflow_logging=True, **automl_settings)
+
+ children = self._get_child_runs(parent)
+ assert len(children) >= 1, "Expected at least 1 child run, got {}".format(len(children))
+
+ @staticmethod
+ def _get_child_runs(parent_run: mlflow.entities.Run) -> DataFrame:
+ experiment_id = parent_run.info.experiment_id
+ return mlflow.search_runs(
+ [experiment_id], filter_string="tags.mlflow.parentRunId = '{}'".format(parent_run.info.run_id)
+ )
+
+ @pytest.fixture(scope="class")
+ def automl_settings(self):
+ return {
+ "time_budget": 2, # in seconds
+ "metric": "accuracy",
+ "task": "classification",
+ "log_file_name": "iris.log",
+ }
diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.md
index 453cdd7ca..28b5ce255 100644
--- a/website/docs/Use-Cases/Task-Oriented-AutoML.md
+++ b/website/docs/Use-Cases/Task-Oriented-AutoML.md
@@ -477,6 +477,18 @@ with mlflow.start_run():
automl.fit(X_train=X_train, y_train=y_train, **settings)
```
+To disable mlflow logging pre-configured in FLAML, set `mlflow_logging=False`:
+```python
+automl = AutoML(mlflow_logging=False)
+```
+or
+```python
+automl.fit(X_train=X_train, y_train=y_train, mlflow_logging=False, **settings)
+```
+
+Setting `mlflow_logging=False` in the constructor will disable mlflow logging for all the `fit()` calls.
+Setting `mlflow_logging=False` in `fit()` will disable mlflow logging for that `fit()` call only.
+
### Extra fit arguments
Extra fit arguments that are needed by the estimators can be passed to `AutoML.fit()`. For example, if there is a weight associated with each training example, they can be passed via `sample_weight`. For another example, `period` can be passed for time series forecaster. For any extra keywork argument passed to `AutoML.fit()` which has not been explicitly listed in the function signature, it will be passed to the underlying estimators' `fit()` as is. For another example, you can set the number of gpus used by each trial with the `gpu_per_trial` argument, which is only used by TransformersEstimator and XGBoostSklearnEstimator.