A programming framework for agentic AI
allow evaluated_rewards shorter than points_to_evaluate (#522)
Chi Wanga1c49ca27be8
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex da1913c5b..f2cedfd15 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -79,8 +79,8 @@ class BlendSearch(Searcher):parameters passed in as points_to_evaluate you can avoidre-running those trials by passing in the reward attributesas a list so the optimiser can be told the results without- needing to re-compute the trial. Must be the same length as- points_to_evaluate.+ needing to re-compute the trial. Must be the same or shorter length than+ points_to_evaluate. When provided, `mode` must be specified.time_budget_s: int or float | Time budget in seconds.num_samples: int | The number of configs to try.resource_attr: A string to specify the resource dimension and the best@@ -115,9 +115,14 @@ class BlendSearch(Searcher):"'low_cost_partial_config'. More info can be found at ""https://microsoft.github.io/FLAML/docs/FAQ#about-low_cost_partial_config-in-tune")- if evaluated_rewards and mode:+ if evaluated_rewards:+ assert mode, "mode must be specified when evaluted_rewards is provided."self._points_to_evaluate = []self._evaluated_rewards = []+ n = len(evaluated_rewards)+ self._evaluated_points = points_to_evaluate[:n]+ new_points_to_evaluate = points_to_evaluate[n:]+ self._all_rewards = evaluated_rewardsbest = max(evaluated_rewards) if mode == "max" else min(evaluated_rewards)# only keep the best points as start pointsfor i, r in enumerate(evaluated_rewards):@@ -125,6 +130,7 @@ class BlendSearch(Searcher):p = points_to_evaluate[i]self._points_to_evaluate.append(p)self._evaluated_rewards.append(r)+ self._points_to_evaluate.extend(new_points_to_evaluate)else:self._points_to_evaluate = points_to_evaluate or []self._evaluated_rewards = evaluated_rewards or []@@ -178,7 +184,7 @@ class BlendSearch(Searcher):mode=mode,seed=gs_seed,sampler=sampler,- points_to_evaluate=points_to_evaluate,+ points_to_evaluate=self._evaluated_points,evaluated_rewards=evaluated_rewards,)except (AssertionError, ValueError):@@ -305,7 +311,26 @@ class BlendSearch(Searcher):)self._gs_admissible_min = self._ls_bound_min.copy()self._gs_admissible_max = self._ls_bound_max.copy()- self._result = {} # config_signature: tuple -> result: Dict+ # config_signature: tuple -> result: Dict+ self._result = (+ {+ self._ls.config_signature(+ *self._ls.complete_config(+ self._evaluated_points[i],+ self._ls_bound_min,+ self._ls_bound_max,+ )+ ): {+ self._metric: r,+ self.cost_attr: 1,+ "config": self._evaluated_points[i],+ }+ for i, r in enumerate(self._all_rewards)+ }+ if self._evaluated_rewards # store all the evaluated rewards+ else {}+ )+if self._metric_constraints:self._metric_constraint_satisfied = Falseself._metric_constraint_penalty = [@@ -708,8 +733,8 @@ class BlendSearch(Searcher):config, space = self._ls.complete_config(init_config, self._ls_bound_min, self._ls_bound_max)+ config_signature = self._ls.config_signature(config, space)if reward is None:- config_signature = self._ls.config_signature(config, space)result = self._result.get(config_signature)if result: # tried beforereturn None@@ -722,7 +747,8 @@ class BlendSearch(Searcher):self._search_thread_pool[0].running += 1self._subspace[trial_id] = spaceif reward is not None:- result = {self._metric: reward, self.cost_attr: 1, "config": config}+ # result = {self._metric: reward, self.cost_attr: 1, "config": config}+ result = self._result[config_signature]self.on_trial_complete(trial_id, result)return Noneif self._use_incumbent_result_in_evaluation:@@ -852,6 +878,15 @@ class BlendSearch(Searcher):return Falsereturn True+ @property+ def results(self) -> List[Dict]:+ """A list of dicts of results for each evaluated configuration.++ Each dict has "config" and metric names as keys.+ The returned dict includes the initial results provided via `evaluated_reward`.+ """+ return [x for x in getattr(self, "_result", {}).values() if x]+try:from ray import __version__ as ray_versiondiff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex 8044f5f51..f281cb81a 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -201,7 +201,7 @@ def run(parameters passed in as points_to_evaluate you can avoidre-running those trials by passing in the reward attributesas a list so the optimiser can be told the results without- needing to re-compute the trial. Must be the same length as+ needing to re-compute the trial. Must be the same or shorter length thanpoints_to_evaluate.e.g.,@@ -210,12 +210,11 @@ def run({"b": .99, "cost_related": {"a": 3}},{"b": .99, "cost_related": {"a": 2}},]- evaluated_rewards=[3.0, 1.0]+ evaluated_rewards = [3.0]```- means that you know the reward for the two configs in- points_to_evaluate are 3.0 and 1.0 respectively and want to- inform run().+ means that you know the reward for the first config in+ points_to_evaluate is 3.0 and want to inform run().resource_attr: A string to specify the resource dimension used bythe scheduler via "scheduler".diff --git a/test/tune/test_tune.py b/test/tune/test_tune.pyindex f816f6adb..9aa14bedf 100644--- a/test/tune/test_tune.py+++ b/test/tune/test_tune.py@@ -223,26 +223,27 @@ def test_nested():logger.info(f"CFO best config: {best_trial.config}")logger.info(f"CFO best result: {best_trial.last_result}")+ bs = BlendSearch(+ experimental=True,+ space=search_space,+ metric="obj",+ mode="min",+ low_cost_partial_config={"cost_related": {"a": 1}},+ points_to_evaluate=[+ {"b": 0.99, "cost_related": {"a": 3}},+ {"b": 0.99, "cost_related": {"a": 2}},+ {"cost_related": {"a": 8}},+ ],+ metric_constraints=[("ab", "<=", 4)],+ )analysis = tune.run(simple_func,- search_alg=BlendSearch(- experimental=True,- space=search_space,- metric="obj",- mode="min",- low_cost_partial_config={"cost_related": {"a": 1}},- points_to_evaluate=[- {"b": 0.99, "cost_related": {"a": 3}},- {"b": 0.99, "cost_related": {"a": 2}},- {"cost_related": {"a": 8}},- ],- metric_constraints=[("ab", "<=", 4)],- ),+ search_alg=bs,local_dir="logs/",num_samples=-1,time_budget_s=1,)-+ print(bs.results)best_trial = analysis.get_best_trial()logger.info(f"BlendSearch exp best config: {best_trial.config}")logger.info(f"BlendSearch exp best result: {best_trial.last_result}")@@ -250,6 +251,7 @@ def test_nested():points_to_evaluate = [{"b": 0.99, "cost_related": {"a": 3}},{"b": 0.99, "cost_related": {"a": 2}},+ {"cost_related": {"a": 8}},]analysis = tune.run(simple_func,@@ -259,7 +261,7 @@ def test_nested():evaluated_rewards=[(config["cost_related"]["a"] - 4) ** 2+ (config["b"] - config["cost_related"]["a"]) ** 2- for config in points_to_evaluate+ for config in points_to_evaluate[:-1]],metric="obj",mode="min",