Skip to content

A programming framework for agentic AI

pred_time_limit clarification and logging (#319)

Chi Wang18230ed22f9c
diff --git a/flaml/automl.py b/flaml/automl.py
index a0e173409..eb084c1e3 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -495,6 +495,7 @@ class AutoML(BaseEstimator):
metric for each model.
mem_thres: A float of the memory size constraint in bytes.
pred_time_limit: A float of the prediction latency constraint in seconds.
+ It refers to the average prediction time per row in validation data.
train_time_limit: A float of the training time constraint in seconds.
verbose: int, default=3 | Controls the verbosity, higher means more
messages.
@@ -1751,6 +1752,7 @@ class AutoML(BaseEstimator):
metric for each model.
mem_thres: A float of the memory size constraint in bytes.
pred_time_limit: A float of the prediction latency constraint in seconds.
+ It refers to the average prediction time per row in validation data.
train_time_limit: A float of the training time constraint in seconds.
X_val: None or a numpy array or a pandas dataframe of validation data.
y_val: None or a numpy array or a pandas series of validation labels.
diff --git a/flaml/ml.py b/flaml/ml.py
index 0441691b4..8293455a1 100644
--- a/flaml/ml.py
+++ b/flaml/ml.py
@@ -206,7 +206,7 @@ def _eval_estimator(
val_loss = sklearn_metric_loss_score(
eval_metric, val_pred_y, y_val, labels, weight_val, groups_val
)
- metric_for_logging = {}
+ metric_for_logging = {"pred_time": pred_time}
if log_training_metric:
train_pred_y = get_y_pred(estimator, X_train, eval_metric, obj)
metric_for_logging["train_loss"] = sklearn_metric_loss_score(
diff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.py
index 7ef672296..515d569b1 100644
--- a/test/automl/test_notebook_example.py
+++ b/test/automl/test_notebook_example.py
@@ -1,4 +1,5 @@
from openml.exceptions import OpenMLServerException
+from requests.exceptions import ChunkedEncodingError
def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):
@@ -8,8 +9,8 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None):
X_train, X_test, y_train, y_test = load_openml_dataset(
dataset_id=1169, data_dir="test/", dataset_format=dataset_format
)
- except OpenMLServerException:
- print("OpenMLServerException raised")
+ except (OpenMLServerException, ChunkedEncodingError) as e:
+ print(e)
return
""" import AutoML class from flaml package """
from flaml import AutoML
@@ -84,8 +85,8 @@ def test_mlflow():
X_train, X_test, y_train, y_test = load_openml_task(
task_id=7592, data_dir="test/"
)
- except OpenMLServerException:
- print("OpenMLServerException raised")
+ except (OpenMLServerException, ChunkedEncodingError) as e:
+ print(e)
return
""" import AutoML class from flaml package """
from flaml import AutoML