Skip to content

A programming framework for agentic AI

support string alg in tune (#1093)

Shaokun7a6414867601
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index 37a91774e..39d612e0a 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -344,7 +344,7 @@ def run(
# do cleanup operation here
return
```
- search_alg: An instance of BlendSearch as the search algorithm
+ search_alg: An instance/string of the search algorithm
to be used. The same instance can be used for iterative tuning.
e.g.,
@@ -481,12 +481,25 @@ def run(
else:
logger.setLevel(logging.CRITICAL)
- from .searcher.blendsearch import BlendSearch, CFO
+ from .searcher.blendsearch import BlendSearch, CFO, RandomSearch
if lexico_objectives is not None:
- logger.warning("If lexico_objectives is not None, search_alg is forced to be CFO")
- search_alg = None
- if search_alg is None:
+ if "modes" not in lexico_objectives.keys():
+ lexico_objectives["modes"] = ["min"] * len(lexico_objectives["metrics"])
+ for t_metric, t_mode in zip(lexico_objectives["metrics"], lexico_objectives["modes"]):
+ if t_metric not in lexico_objectives["tolerances"].keys():
+ lexico_objectives["tolerances"][t_metric] = 0
+ if t_metric not in lexico_objectives["targets"].keys():
+ lexico_objectives["targets"][t_metric] = -float("inf") if t_mode == "min" else float("inf")
+ if search_alg is None or isinstance(search_alg, str):
+ if isinstance(search_alg, str):
+ assert search_alg in [
+ "BlendSearch",
+ "CFO",
+ "CFOCat",
+ "RandomSearch",
+ ], f"search_alg={search_alg} is not recognized. 'BlendSearch', 'CFO', 'CFOcat' and 'RandomSearch' are supported."
+
flaml_scheduler_resource_attr = (
flaml_scheduler_min_resource
) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None
@@ -500,20 +513,30 @@ def run(
flaml_scheduler_max_resource = max_resource
flaml_scheduler_reduction_factor = reduction_factor
scheduler = None
- if lexico_objectives is None:
- try:
- import optuna as _
+ if lexico_objectives:
+ # TODO: Modify after supporting BlendSearch in lexicographic optimization
+ SearchAlgorithm = CFO
+ logger.info(
+ f"Using search algorithm {SearchAlgorithm.__name__} for lexicographic optimization. Note that when providing other search algorithms, we use CFO instead temporarily."
+ )
+ metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
+ else:
+ if not search_alg or search_alg == "BlendSearch":
+ try:
+ import optuna as _
- SearchAlgorithm = BlendSearch
+ SearchAlgorithm = BlendSearch
+ logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
+ except ImportError:
+ if search_alg == "BlendSearch":
+ raise ValueError("To use BlendSearch, run: pip install flaml[blendsearch]")
+ else:
+ SearchAlgorithm = CFO
+ logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
+ else:
+ SearchAlgorithm = locals()[search_alg]
logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
- except ImportError:
- SearchAlgorithm = CFO
- logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
metric = metric or DEFAULT_METRIC
- else:
- SearchAlgorithm = CFO
- logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
- metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
search_alg = SearchAlgorithm(
metric=metric,
mode=mode,
@@ -535,8 +558,12 @@ def run(
)
else:
if metric is None or mode is None:
- metric = metric or search_alg.metric or DEFAULT_METRIC
- mode = mode or search_alg.mode
+ if lexico_objectives:
+ metric = lexico_objectives["metrics"][0] or metric or search_alg.metric or DEFAULT_METRIC
+ mode = lexico_objectives["modes"][0] or mode or search_alg.mode
+ else:
+ metric = metric or search_alg.metric or DEFAULT_METRIC
+ mode = mode or search_alg.mode
if ray_available and use_ray:
if ray_version.startswith("1."):
from ray.tune.suggest import ConcurrencyLimiter
@@ -555,6 +582,13 @@ def run(
):
search_alg.use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
searcher = search_alg.searcher if isinstance(search_alg, ConcurrencyLimiter) else search_alg
+ if lexico_objectives:
+ # TODO: Modify after supporting BlendSearch in lexicographic optimization
+ assert search_alg.__class__.__name__ in [
+ "CFO",
+ ], "If lexico_objectives is not None, the search_alg must be CFO for now."
+ search_alg.lexico_objective = lexico_objectives
+
if isinstance(searcher, BlendSearch):
setting = {}
if time_budget_s:
diff --git a/setup.py b/setup.py
index dddba08f8..a5c5de225 100644
--- a/setup.py
+++ b/setup.py
@@ -85,7 +85,7 @@ setuptools.setup(
"tensorboardX==2.6", # test_forecast_panel
"requests<2.29.0", # https://github.com/docker/docker-py/issues/3113
"packaging",
- "pydantic",
+ "pydantic==1.10.9",
"sympy",
"wolframalpha",
"joblib<1.3.0", # temp solution for joblib 1.3.0 issue, no need once https://github.com/joblib/joblib-spark/pull/48 is merged
@@ -137,7 +137,7 @@ setuptools.setup(
"benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3", "pandas==1.1.4"],
"openai": ["openai==0.27.8", "diskcache"],
"autogen": ["openai==0.27.8", "diskcache", "docker"],
- "mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic", "wolframalpha"],
+ "mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic==1.10.9", "wolframalpha"],
"synapse": [
"joblibspark>=0.5.0",
"optuna==2.8.0",
diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py
index 8ada5519a..7dec2df08 100644
--- a/test/tune/test_tune.py
+++ b/test/tune/test_tune.py
@@ -1,6 +1,6 @@
"""Require: pip install flaml[test,ray]
"""
-from flaml import BlendSearch
+from flaml import BlendSearch, CFO
import time
import os
from sklearn.model_selection import train_test_split
@@ -8,6 +8,7 @@ import sklearn.metrics
import sklearn.datasets
import xgboost as xgb
import logging
+import math
try:
from ray.tune.integration.xgboost import TuneReportCheckpointCallback
@@ -20,6 +21,32 @@ logger.addHandler(logging.FileHandler("logs/tune.log"))
logger.setLevel(logging.INFO)
+def _BraninCurrin(config):
+ # Rescale brain
+ x_1 = 15 * config["x1"] - 5
+ x_2 = 15 * config["x2"]
+ # Brain function
+ t1 = x_2 - 5.1 / (4 * math.pi**2) * x_1**2 + 5 / math.pi * x_1 - 6
+ t2 = 10 * (1 - 1 / (8 * math.pi)) * math.cos(x_1)
+ brain_result = t1**2 + t2 + 10
+ # Currin function
+ xc_1 = config["x1"]
+ xc_2 = config["x2"]
+ factor1 = 1 - math.exp(-1 / (2 * xc_2))
+ numer = 2300 * pow(xc_1, 3) + 1900 * pow(xc_1, 2) + 2092 * xc_1 + 60
+ denom = 100 * pow(xc_1, 3) + 500 * pow(xc_1, 2) + 4 * xc_1 + 20
+ currin_result = factor1 * numer / denom
+ return {"brain": brain_result, "currin": currin_result}
+
+
+def _easy_objective(config):
+ # Hyperparameters
+ width, height, step = config["width"], config["height"], config["steps"]
+
+ # get_result
+ return {"mean_loss": (0.1 + width * step / 100) ** (-1) + height * 0.1}
+
+
def test_nested_run():
from flaml import AutoML, tune
@@ -352,6 +379,72 @@ def test_run_training_function_return_value():
)
+def test_passing_search_alg():
+ from flaml import tune
+
+ # search_space
+ so_search_space = {
+ "steps": 100,
+ "width": tune.uniform(0, 20),
+ "height": tune.uniform(-100, 100),
+ }
+ mo_search_space = {
+ "x1": tune.uniform(lower=0.000001, upper=1.0),
+ "x2": tune.uniform(lower=0.000001, upper=1.0),
+ }
+
+ # lexicographic objectives
+ lexico_objectives = {}
+ lexico_objectives["metrics"] = ["brain", "currin"]
+ lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
+ lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
+ lexico_objectives["modes"] = ["min", "min"]
+
+ ## Passing search_alg through string
+ # Non lexico tune
+ tune.run(
+ _easy_objective,
+ search_alg="BlendSearch",
+ metric="mean_loss",
+ mode="min",
+ num_samples=10,
+ config=so_search_space,
+ )
+ # lexico tune
+ tune.run(
+ _BraninCurrin, search_alg="CFO", num_samples=10, config=mo_search_space, lexico_objectives=lexico_objectives
+ )
+ tune.run(
+ _BraninCurrin,
+ search_alg="BlendSearch",
+ num_samples=10,
+ config=mo_search_space,
+ lexico_objectives=lexico_objectives,
+ )
+
+ ## Passing search_alg through instance
+ so_bs = BlendSearch(time_budget_s=5, metric="mean_loss", mode="min")
+ # TODO: We will change CFO into blendsearch in the future
+ mo_bs = CFO(time_budget_s=5)
+ # Non lexico tune
+ tune.run(
+ _easy_objective,
+ search_alg=so_bs,
+ metric="mean_loss",
+ mode="min",
+ num_samples=10,
+ config=so_search_space,
+ )
+ # lexico tune
+ tune.run(
+ _BraninCurrin,
+ search_alg=mo_bs,
+ num_samples=10,
+ config=mo_search_space,
+ lexico_objectives=lexico_objectives,
+ )
+
+
def test_xgboost_bs():
_test_xgboost()