A programming framework for agentic AI
example update (#359)
Chi Wang2f5d6169d3b5
diff --git a/flaml/automl.py b/flaml/automl.pyindex 8df3543eb..1ec2cdcd2 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -433,10 +433,8 @@ class AutoML(BaseEstimator):):return metric_to_minimize, metrics_to_log```-which returns a float number as the minimization objective,and a dictionary as the metrics to log. E.g.,-```pythondef custom_metric(X_val, y_val, estimator, labels,@@ -468,7 +466,6 @@ class AutoML(BaseEstimator):set it to be an empty string "".estimator_list: A list of strings for estimator names, or 'auto'e.g., ```['lgbm', 'xgboost', 'xgb_limitdepth', 'catboost', 'rf', 'extra_tree']```-time_budget: A float number of the time budget in seconds.Use -1 if no time limit.max_iter: An integer of the maximal number of iterations.@@ -531,7 +528,6 @@ class AutoML(BaseEstimator):`automl` object and use them in the `new_automl` object.e.g.,-```pythonfrom flaml import AutoMLautoml = AutoML()@@ -1717,7 +1713,6 @@ class AutoML(BaseEstimator):'mape'. Default is 'auto'.If passing a customized metric function, the function needs tohave the follwing signature:-```pythondef custom_metric(X_test, y_test, estimator, labels,@@ -1726,33 +1721,30 @@ class AutoML(BaseEstimator):):return metric_to_minimize, metrics_to_log```-which returns a float number as the minimization objective,and a dictionary as the metrics to log. E.g.,+ ```python+ def custom_metric(+ X_val, y_val, estimator, labels,+ X_train, y_train, weight_val=None, weight_train=None,+ **args,+ ):+ from sklearn.metrics import log_loss+ import time- .. code-block:: python-- def custom_metric(- X_val, y_val, estimator, labels,- X_train, y_train, weight_val=None, weight_train=None,- **args,- ):- from sklearn.metrics import log_loss- import time-- start = time.time()- y_pred = estimator.predict_proba(X_val)- pred_time = (time.time() - start) / len(X_val)- val_loss = log_loss(y_val, y_pred, labels=labels, sample_weight=weight_val)- y_pred = estimator.predict_proba(X_train)- train_loss = log_loss(y_train, y_pred, labels=labels, sample_weight=weight_train)- alpha = 0.5- return val_loss * (1 + alpha) - alpha * train_loss, {- "val_loss": val_loss,- "train_loss": train_loss,- "pred_time": pred_time,- }-+ start = time.time()+ y_pred = estimator.predict_proba(X_val)+ pred_time = (time.time() - start) / len(X_val)+ val_loss = log_loss(y_val, y_pred, labels=labels, sample_weight=weight_val)+ y_pred = estimator.predict_proba(X_train)+ train_loss = log_loss(y_train, y_pred, labels=labels, sample_weight=weight_train)+ alpha = 0.5+ return val_loss * (1 + alpha) - alpha * train_loss, {+ "val_loss": val_loss,+ "train_loss": train_loss,+ "pred_time": pred_time,+ }+ ```task: A string of the task type, e.g.,'classification', 'regression', 'ts_forecast', 'rank','seq-classification', 'seq-regression', 'summarization'diff --git a/test/automl/test_classification.py b/test/automl/test_classification.pyindex 29507e08f..66d8cdfd2 100644--- a/test/automl/test_classification.py+++ b/test/automl/test_classification.py@@ -2,6 +2,7 @@ import unittestimport numpy as npimport scipy.sparsefrom sklearn.datasets import load_breast_cancer+from sklearn.model_selection import train_test_splitimport pandas as pdfrom datetime import datetimefrom flaml import AutoML@@ -221,14 +222,28 @@ class TestClassification(unittest.TestCase):print(automl_experiment.best_estimator)def test_ray_classification(self):- from sklearn.datasets import make_classification+ X, y = load_breast_cancer(return_X_y=True)+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)- X, y = make_classification(1000, 10)automl = AutoML()try:- automl.fit(X, y, time_budget=10, task="classification", use_ray=True)automl.fit(- X, y, time_budget=10, task="classification", n_concurrent_trials=2+ X_train,+ y_train,+ X_val=X_test,+ y_val=y_test,+ time_budget=10,+ task="classification",+ use_ray=True,+ )+ automl.fit(+ X_train,+ y_train,+ X_val=X_test,+ y_val=y_test,+ time_budget=10,+ task="classification",+ n_concurrent_trials=2,)except ImportError:returndiff --git a/test/ray/distribute_tune.py b/test/ray/distribute_tune.pyindex 04120233b..ee3580d0b 100644--- a/test/ray/distribute_tune.py+++ b/test/ray/distribute_tune.py@@ -1,30 +1,28 @@import rayimport lightgbm as lgbimport numpy as np-import sklearn.datasets-import sklearn.metrics+from sklearn.datasets import load_breast_cancer+from sklearn.metrics import accuracy_scorefrom sklearn.model_selection import train_test_splitfrom flaml import tunefrom flaml.model import LGBMEstimator-data, target = sklearn.datasets.load_breast_cancer(return_X_y=True)-train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.25)+X, y = load_breast_cancer(return_X_y=True)+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)def train_breast_cancer(config):params = LGBMEstimator(**config).params- train_set = lgb.Dataset(train_x, label=train_y)+ train_set = lgb.Dataset(X_train, label=y_train)gbm = lgb.train(params, train_set)- preds = gbm.predict(test_x)+ preds = gbm.predict(X_test)pred_labels = np.rint(preds)- tune.report(- mean_accuracy=sklearn.metrics.accuracy_score(test_y, pred_labels), done=True- )+ tune.report(mean_accuracy=accuracy_score(y_test, pred_labels), done=True)if __name__ == "__main__":ray.init(address="auto")- flaml_lgbm_search_space = LGBMEstimator.search_space(train_x.shape)+ flaml_lgbm_search_space = LGBMEstimator.search_space(X_train.shape)config_search_space = {hp: space["domain"] for hp, space in flaml_lgbm_search_space.items()}diff --git a/test/tune.py b/test/tune_example.pysimilarity index 87%rename from test/tune.pyrename to test/tune_example.pyindex 8b67b2b31..012c70945 100644--- a/test/tune.py+++ b/test/tune_example.py@@ -36,6 +36,14 @@ low_cost_partial_config = {for hp, space in flaml_lgbm_search_space.items()if "low_cost_init_value" in space}+# initial points to evaluate+points_to_evaluate = [+ {+ hp: space["init_value"]+ for hp, space in flaml_lgbm_search_space.items()+ if "init_value" in space+ }+]# run the tuning, minimizing mse, with total time budget 3 secondsanalysis = tune.run(train_lgbm,@@ -43,6 +51,7 @@ analysis = tune.run(mode="min",config=config_search_space,low_cost_partial_config=low_cost_partial_config,+ points_to_evaluate=points_to_evaluate,time_budget_s=3,num_samples=-1,)diff --git a/website/docs/Getting-Started.md b/website/docs/Getting-Started.mdindex f327251ca..755384abf 100644--- a/website/docs/Getting-Started.md+++ b/website/docs/Getting-Started.md@@ -74,7 +74,7 @@ analysis = tune.run(low_cost_partial_config=low_cost_partial_config, time_budget_s=3, num_samples=-1,)```-Please see this [script](https://github.com/microsoft/FLAML/blob/main/test/tune.py) for the complete version of the above example.+Please see this [script](https://github.com/microsoft/FLAML/blob/main/test/tune_example.py) for the complete version of the above example.### Where to Go Next?