A programming framework for agentic AI
add consistency test (#216)
Qingyun Wub1115d5347e0
diff --git a/flaml/tune/analysis.py b/flaml/tune/analysis.pyindex bd8135b94..320b0d279 100644--- a/flaml/tune/analysis.py+++ b/flaml/tune/analysis.py@@ -1,4 +1,4 @@-'''+"""Copyright 2020 The Ray Authors.Licensed under the Apache License, Version 2.0 (the "License");@@ -16,12 +16,13 @@ limitations under the License.This source file is adapted here because ray does not fully support Windows.Copyright (c) Microsoft Corporation.-'''+"""from typing import Dict, Optionalimport numpy as npfrom .trial import Trialimport logging+logger = logging.getLogger(__name__)@@ -30,8 +31,7 @@ def is_nan_or_inf(value):class ExperimentAnalysis:- """Analyze results from a Tune experiment.- """+ """Analyze results from a Tune experiment."""@propertydef best_trial(self) -> Trial:@@ -46,7 +46,8 @@ class ExperimentAnalysis:"To fetch the `best_trial`, pass a `metric` and `mode` ""parameter to `tune.run()`. Alternatively, use the ""`get_best_trial(metric, mode)` method to set the metric "- "and mode explicitly.")+ "and mode explicitly."+ )return self.get_best_trial(self.default_metric, self.default_mode)@property@@ -62,30 +63,41 @@ class ExperimentAnalysis:"To fetch the `best_config`, pass a `metric` and `mode` ""parameter to `tune.run()`. Alternatively, use the ""`get_best_config(metric, mode)` method to set the metric "- "and mode explicitly.")+ "and mode explicitly."+ )return self.get_best_config(self.default_metric, self.default_mode)+ @property+ def results(self) -> Dict[str, Dict]:+ """Get the last result of all the trials of the experiment"""++ return {trial.trial_id: trial.last_result for trial in self.trials}+def _validate_metric(self, metric: str) -> str:if not metric and not self.default_metric:raise ValueError("No `metric` has been passed and `default_metric` has "- "not been set. Please specify the `metric` parameter.")+ "not been set. Please specify the `metric` parameter."+ )return metric or self.default_metricdef _validate_mode(self, mode: str) -> str:if not mode and not self.default_mode:raise ValueError("No `mode` has been passed and `default_mode` has "- "not been set. Please specify the `mode` parameter.")+ "not been set. Please specify the `mode` parameter."+ )if mode and mode not in ["min", "max"]:raise ValueError("If set, `mode` has to be one of [min, max]")return mode or self.default_mode- def get_best_trial(self,- metric: Optional[str] = None,- mode: Optional[str] = None,- scope: str = "last",- filter_nan_and_inf: bool = True) -> Optional[Trial]:+ def get_best_trial(+ self,+ metric: Optional[str] = None,+ mode: Optional[str] = None,+ scope: str = "last",+ filter_nan_and_inf: bool = True,+ ) -> Optional[Trial]:"""Retrieve the best trial object.Compares all trials' scores on ``metric``.If ``metric`` is not specified, ``self.default_metric`` will be used.@@ -116,11 +128,13 @@ class ExperimentAnalysis:if scope not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]:raise ValueError("ExperimentAnalysis: attempting to get best trial for "- "metric {} for scope {} not in [\"all\", \"last\", \"avg\", "- "\"last-5-avg\", \"last-10-avg\"]. "+ 'metric {} for scope {} not in ["all", "last", "avg", '+ '"last-5-avg", "last-10-avg"]. '"If you didn't pass a `metric` parameter to `tune.run()`, ""you have to pass one when fetching the best trial.".format(- metric, scope))+ metric, scope+ )+ )best_trial = Nonebest_metric_score = Nonefor trial in self.trials:@@ -150,13 +164,16 @@ class ExperimentAnalysis:if not best_trial:logger.warning("Could not find best trial. Did you pass the correct `metric` "- "parameter?")+ "parameter?"+ )return best_trial- def get_best_config(self,- metric: Optional[str] = None,- mode: Optional[str] = None,- scope: str = "last") -> Optional[Dict]:+ def get_best_config(+ self,+ metric: Optional[str] = None,+ mode: Optional[str] = None,+ scope: str = "last",+ ) -> Optional[Dict]:"""Retrieve the best config corresponding to the trial.Compares all trials' scores on `metric`.If ``metric`` is not specified, ``self.default_metric`` will be used.diff --git a/test/tune/test_flaml_raytune_consistency.py b/test/tune/test_flaml_raytune_consistency.pynew file mode 100644index 000000000..dee393c3a--- /dev/null+++ b/test/tune/test_flaml_raytune_consistency.py@@ -0,0 +1,123 @@+# import unittest+import numpy as np++# require: pip install flaml[blendsearch, ray]+# require: pip install flaml[ray]+import time+from flaml import tune+++def evaluate_config(config):+ """evaluate a hyperparameter configuration"""+ # we uss a toy example with 2 hyperparameters+ metric = (round(config["x"]) - 85000) ** 2 - config["x"] / config["y"]+ # usually the evaluation takes an non-neglible cost+ # and the cost could be related to certain hyperparameters+ # in this example, we assume it's proportional to x+ time.sleep(config["x"] / 100000)+ # use tune.report to report the metric to optimize+ tune.report(metric=metric)+++config_search_space = {+ "x": tune.lograndint(lower=1, upper=100000),+ "y": tune.randint(lower=1, upper=100000),+}++low_cost_partial_config = {"x": 1}+++def setup_searcher(searcher_name):+ from flaml.searcher.blendsearch import BlendSearch, CFO, RandomSearch++ if "cfo" in searcher_name:+ searcher = CFO(+ space=config_search_space, low_cost_partial_config=low_cost_partial_config+ )+ elif searcher_name == "bs":+ searcher = BlendSearch(+ metric="metric",+ mode="min",+ space=config_search_space,+ low_cost_partial_config=low_cost_partial_config,+ )+ elif searcher_name == "random":+ searcher = RandomSearch(space=config_search_space)+ else:+ return None+ return searcher+++def _test_flaml_raytune_consistency(+ num_samples=-1, max_concurrent_trials=1, searcher_name="cfo"+):+ try:+ from ray import tune as raytune+ except ImportError:+ print(+ "skip _test_flaml_raytune_consistency because ray tune cannot be imported."+ )+ return+ np.random.seed(100)+ searcher = setup_searcher(searcher_name)+ analysis = tune.run(+ evaluate_config, # the function to evaluate a config+ config=config_search_space, # the search space+ low_cost_partial_config=low_cost_partial_config, # a initial (partial) config with low cost+ metric="metric", # the name of the metric used for optimization+ mode="min", # the optimization mode, 'min' or 'max'+ num_samples=num_samples, # the maximal number of configs to try, -1 means infinite+ time_budget_s=None, # the time budget in seconds+ local_dir="logs/", # the local directory to store logs+ search_alg=searcher,+ # verbose=0, # verbosity+ # use_ray=True, # uncomment when performing parallel tuning using ray+ )+ flaml_best_config = analysis.best_config+ flaml_config_in_results = [v["config"] for v in analysis.results.values()]+ print(analysis.best_trial.last_result) # the best trial's result+ print("best flaml", searcher_name, flaml_best_config) # the best config+ print("flaml config in results", searcher_name, flaml_config_in_results)++ np.random.seed(100)+ searcher = setup_searcher(searcher_name)+ from ray.tune.suggest import ConcurrencyLimiter++ search_alg = ConcurrencyLimiter(searcher, max_concurrent_trials)+ analysis = raytune.run(+ evaluate_config, # the function to evaluate a config+ config=config_search_space,+ metric="metric", # the name of the metric used for optimization+ mode="min", # the optimization mode, 'min' or 'max'+ num_samples=num_samples, # the maximal number of configs to try, -1 means infinite+ local_dir="logs/", # the local directory to store logs+ # max_concurrent_trials=max_concurrent_trials,+ # resources_per_trial={"cpu": max_concurrent_trials, "gpu": 0},+ search_alg=search_alg,+ )+ ray_best_config = analysis.best_config+ ray_config_in_results = [v["config"] for v in analysis.results.values()]+ print(analysis.best_trial.last_result) # the best trial's result+ print("ray best", searcher_name, analysis.best_config) # the best config+ print("ray config in results", searcher_name, ray_config_in_results)+ assert ray_best_config == flaml_best_config, "best config should be the same"+ assert (+ flaml_config_in_results == ray_config_in_results+ ), "results from raytune and flaml should be the same"+++def test_consistency():+ _test_flaml_raytune_consistency(+ num_samples=5, max_concurrent_trials=1, searcher_name="random"+ )+ _test_flaml_raytune_consistency(+ num_samples=5, max_concurrent_trials=1, searcher_name="cfo"+ )+ _test_flaml_raytune_consistency(+ num_samples=5, max_concurrent_trials=1, searcher_name="bs"+ )+++if __name__ == "__main__":+ # unittest.main()+ test_consistency()