A programming framework for agentic AI
Makes the evaluation_function could receive the incumbent best result as input in Tune (#339)
Z.sk7b24662dca1b
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex 9de56c203..39aa7f058 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -18,6 +18,7 @@ except (ImportError, AssertionError):from .suggestion import Searcherfrom .suggestion import OptunaSearch as GlobalSearchfrom ..tune.trial import unflatten_dict, flatten_dict+from ..tune import INCUMBENT_RESULTfrom .search_thread import SearchThreadfrom .flow2 import FLOW2from ..tune.space import add_cost_to_space, indexof, normalize, define_by_run_func@@ -56,6 +57,7 @@ class BlendSearch(Searcher):metric_constraints: Optional[List[Tuple[str, str, float]]] = None,seed: Optional[int] = 20,experimental: Optional[bool] = False,+ use_incumbent_result_in_evaluation=False,):"""Constructor.@@ -118,6 +120,7 @@ class BlendSearch(Searcher):experimental: A bool of whether to use experimental features."""self._metric, self._mode = metric, mode+ self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluationinit_config = low_cost_partial_config or {}if not init_config:logger.info(@@ -734,6 +737,12 @@ class BlendSearch(Searcher):result = {self._metric: reward, self.cost_attr: 1, "config": config}self.on_trial_complete(trial_id, result)return None+ if self._use_incumbent_result_in_evaluation:+ if self._trial_proposed_by[trial_id] > 0:+ choice_thread = self._search_thread_pool[+ self._trial_proposed_by[trial_id]+ ]+ config[INCUMBENT_RESULT] = choice_thread.best_resultreturn configdef _should_skip(self, choice, trial_id, config, space) -> bool:diff --git a/flaml/searcher/search_thread.py b/flaml/searcher/search_thread.pyindex 2bad28da5..a148256b8 100644--- a/flaml/searcher/search_thread.py+++ b/flaml/searcher/search_thread.py@@ -42,6 +42,7 @@ class SearchThread:self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf) # inherently minimize+ self.best_result = None# eci: estimated cost for improvementself.eci = self.cost_bestself.priority = self.speed = 0@@ -139,12 +140,13 @@ class SearchThread:self.cost_total += self.cost_lastif self._search_alg.metric in result:obj = result[self._search_alg.metric] * self._metric_op- if obj < self.obj_best1:+ if obj < self.obj_best1 or self.best_result is None:self.cost_best2 = self.cost_best1self.cost_best1 = self.cost_totalself.obj_best2 = obj if np.isinf(self.obj_best1) else self.obj_best1self.obj_best1 = objself.cost_best = self.cost_last+ self.best_result = resultself._update_speed()self.running -= 1assert self.running >= 0diff --git a/flaml/tune/__init__.py b/flaml/tune/__init__.pyindex 111b1a532..755159339 100644--- a/flaml/tune/__init__.py+++ b/flaml/tune/__init__.py@@ -1,12 +1,35 @@try:from ray import __version__ as ray_version- assert ray_version >= '1.0.0'- from ray.tune import (uniform, quniform, choice, randint, qrandint, randn,- qrandn, loguniform, qloguniform, lograndint, qlograndint)++ assert ray_version >= "1.0.0"+ from ray.tune import (+ uniform,+ quniform,+ choice,+ randint,+ qrandint,+ randn,+ qrandn,+ loguniform,+ qloguniform,+ lograndint,+ qlograndint,+ )except (ImportError, AssertionError):- from .sample import (uniform, quniform, choice, randint, qrandint, randn,- qrandn, loguniform, qloguniform, lograndint, qlograndint)-from .tune import run, report+ from .sample import (+ uniform,+ quniform,+ choice,+ randint,+ qrandint,+ randn,+ qrandn,+ loguniform,+ qloguniform,+ lograndint,+ qlograndint,+ )+from .tune import run, report, INCUMBENT_RESULTfrom .sample import polynomial_expansion_setfrom .sample import PolynomialExpansionSet, Categorical, Floatfrom .trial import Trialdiff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex 2c9729c1d..43ab19912 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -30,6 +30,8 @@ _verbose = 0_running_trial = None_training_iteration = 0+INCUMBENT_RESULT = "__incumbent_result__"+class ExperimentAnalysis(EA):"""Class for storing the experiment results."""@@ -96,6 +98,8 @@ def report(_metric=None, **kwargs):_running_trial = trialresult["training_iteration"] = _training_iterationresult["config"] = trial.config+ if INCUMBENT_RESULT in result["config"]:+ del result["config"][INCUMBENT_RESULT]for key, value in trial.config.items():result["config/" + key] = value_runner.process_trial_result(_runner.running_trial, result)@@ -134,6 +138,7 @@ def run(metric_constraints: Optional[List[Tuple[str, str, float]]] = None,max_failure: Optional[int] = 100,use_ray: Optional[bool] = False,+ use_incumbent_result_in_evaluation: Optional[bool] = None,):"""The trigger for HPO.@@ -351,6 +356,7 @@ def run(reduction_factor=flaml_scheduler_reduction_factor,config_constraints=config_constraints,metric_constraints=metric_constraints,+ use_incumbent_result_in_evaluation=use_incumbent_result_in_evaluation,)else:if metric is None or mode is None:@@ -360,6 +366,18 @@ def run(from ray.tune.suggest import ConcurrencyLimiterelse:from flaml.searcher.suggestion import ConcurrencyLimiter+ if (+ search_alg.__class__.__name__+ in [+ "BlendSearch",+ "CFO",+ "CFOCat",+ ]+ and use_incumbent_result_in_evaluation is not None+ ):+ search_alg.use_incumbent_result_in_evaluation = (+ use_incumbent_result_in_evaluation+ )searcher = (search_alg.searcherif isinstance(search_alg, ConcurrencyLimiter)diff --git a/test/tune/test_record_incumbent.py b/test/tune/test_record_incumbent.pynew file mode 100644index 000000000..b267ee0f9--- /dev/null+++ b/test/tune/test_record_incumbent.py@@ -0,0 +1,85 @@+import numpy as np+from flaml import tune+from flaml.tune import INCUMBENT_RESULT+++def rosenbrock_function(config: dict):+ funcLoss = 50+ for key, value in config.items():+ if key in ["x1", "x2", "x3", "x4", "x5"]:+ funcLoss += value ** 2 - 10 * np.cos(2 * np.pi * value)+ if INCUMBENT_RESULT in config.keys():+ print("----------------------------------------------")+ print("incumbent result", config[INCUMBENT_RESULT])+ print("----------------------------------------------")++ return {"funcLoss": funcLoss}+++def test_record_incumbent(method="BlendSearch"):++ if method != "CFOCat":+ search_space = {+ "x1": tune.randint(1, 9),+ "x2": tune.randint(1, 9),+ "x3": tune.randint(1, 9),+ "x4": tune.randint(1, 9),+ "x5": tune.randint(1, 9),+ }+ else:+ search_space = {+ "x1": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),+ "x2": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),+ "x3": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),+ "x4": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),+ "x5": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),+ }++ max_iter = 100+ num_samples = 128+ time_budget_s = 1+ n_cpu = 1++ if method == "BlendSearch":+ tune.run(+ evaluation_function=rosenbrock_function,+ config=search_space,+ verbose=0,+ metric="funcLoss",+ mode="min",+ max_resource=max_iter,+ min_resource=1,+ local_dir="logs/",+ num_samples=num_samples * n_cpu,+ time_budget_s=time_budget_s,+ use_incumbent_result_in_evaluation=True,+ )+ return+ elif method == "CFO":+ from flaml import CFO++ algo = CFO(+ use_incumbent_result_in_evaluation=True,+ )+ elif method == "CFOCat":+ from flaml.searcher.cfo_cat import CFOCat++ algo = CFOCat(+ use_incumbent_result_in_evaluation=True,+ )+ else:+ raise NotImplementedError+ tune.run(+ evaluation_function=rosenbrock_function,+ metric="funcLoss",+ mode="min",+ config=search_space,+ local_dir="logs/",+ num_samples=num_samples * n_cpu,+ time_budget_s=time_budget_s,+ search_alg=algo,+ )+++if __name__ == "__main__":+ test_record_incumbent(method="BlendSearch")