A programming framework for agentic AI
fix a bug when using ray & update ray on aml (#455)
Chi Wang9e88f221679c
diff --git a/flaml/automl.py b/flaml/automl.pyindex f7fa4a130..02915df07 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -213,41 +213,42 @@ class AutoMLState:groups = self.groups_allreturn sampled_X_train, sampled_y_train, sampled_weight, groups- def _compute_with_config_base(self, estimator, config_w_resource):+ @staticmethod+ def _compute_with_config_base(config_w_resource, state, estimator):if "FLAML_sample_size" in config_w_resource:sample_size = int(config_w_resource["FLAML_sample_size"])else:- sample_size = self.data_size[0]+ sample_size = state.data_size[0](sampled_X_train,sampled_y_train,sampled_weight,groups,- ) = self._prepare_sample_train_data(sample_size)+ ) = state._prepare_sample_train_data(sample_size)if sampled_weight is not None:- weight = self.fit_kwargs["sample_weight"]- self.fit_kwargs["sample_weight"] = sampled_weight+ weight = state.fit_kwargs["sample_weight"]+ state.fit_kwargs["sample_weight"] = sampled_weightelse:weight = Noneif groups is not None:- self.fit_kwargs["groups"] = groups+ state.fit_kwargs["groups"] = groupsconfig = config_w_resource.copy()if "FLAML_sample_size" in config:del config["FLAML_sample_size"]budget = (None- if self.time_budget is None- else self.time_budget - self.time_from_start- if sample_size == self.data_size[0]- else (self.time_budget - self.time_from_start)+ if state.time_budget is None+ else state.time_budget - state.time_from_start+ if sample_size == state.data_size[0]+ else (state.time_budget - state.time_from_start)/ 2* sample_size- / self.data_size[0]+ / state.data_size[0])- if _is_nlp_task(self.task):- self.fit_kwargs["X_val"] = self.X_val- self.fit_kwargs["y_val"] = self.y_val+ if _is_nlp_task(state.task):+ state.fit_kwargs["X_val"] = state.X_val+ state.fit_kwargs["y_val"] = state.y_val(trained_estimator,@@ -258,41 +259,42 @@ class AutoMLState:) = compute_estimator(sampled_X_train,sampled_y_train,- self.X_val,- self.y_val,- self.weight_val,- self.groups_val,- self.train_time_limit+ state.X_val,+ state.y_val,+ state.weight_val,+ state.groups_val,+ state.train_time_limitif budget is None- else min(budget, self.train_time_limit),- self.kf,+ else min(budget, state.train_time_limit),+ state.kf,config,- self.task,+ state.task,estimator,- self.eval_method,- self.metric,- self.best_loss,- self.n_jobs,- self.learner_classes.get(estimator),- self.log_training_metric,- self.fit_kwargs,+ state.eval_method,+ state.metric,+ state.best_loss,+ state.n_jobs,+ state.learner_classes.get(estimator),+ state.log_training_metric,+ state.fit_kwargs,)- if self.retrain_final and not self.model_history:+ if state.retrain_final and not state.model_history:trained_estimator.cleanup()- if _is_nlp_task(self.task):- del self.fit_kwargs["X_val"]- del self.fit_kwargs["y_val"]+ if _is_nlp_task(state.task):+ del state.fit_kwargs["X_val"]+ del state.fit_kwargs["y_val"]result = {"pred_time": pred_time,- "wall_clock_time": time.time() - self._start_time_flag,+ "wall_clock_time": time.time() - state._start_time_flag,"metric_for_logging": metric_for_logging,"val_loss": val_loss,"trained_estimator": trained_estimator,}if sampled_weight is not None:- self.fit_kwargs["sample_weight"] = weight+ state.fit_kwargs["sample_weight"] = weight+ tune.report(**result)return resultdef _train_with_config(@@ -1672,12 +1674,14 @@ class AutoML(BaseEstimator):search_state.training_function = with_parameters(AutoMLState._compute_with_config_base,- self=self._state,+ state=self._state,estimator=estimator,)else:search_state.training_function = partial(- AutoMLState._compute_with_config_base, self._state, estimator+ AutoMLState._compute_with_config_base,+ state=self._state,+ estimator=estimator,)states = self._search_statesmem_res = self._mem_thres@@ -2461,7 +2465,9 @@ class AutoML(BaseEstimator):)if not search_state.search_alg:search_state.training_function = partial(- AutoMLState._compute_with_config_base, self._state, estimator+ AutoMLState._compute_with_config_base,+ state=self._state,+ estimator=estimator,)search_space = search_state.search_spaceif self._sample:diff --git a/flaml/tune/trial_runner.py b/flaml/tune/trial_runner.pyindex 3419dc3ec..ec305a8c0 100644--- a/flaml/tune/trial_runner.py+++ b/flaml/tune/trial_runner.py@@ -125,3 +125,7 @@ class SequentialTrialRunner(BaseTrialRunner):trial = Noneself.running_trial = trialreturn trial++ def stop_trial(self, trial):+ super().stop_trial(trial)+ self.running_trial = Nonediff --git a/flaml/tune/tune.py b/flaml/tune/tune.pyindex 396f83a2e..61f400c25 100644--- a/flaml/tune/tune.py+++ b/flaml/tune/tune.py@@ -90,7 +90,9 @@ def report(_metric=None, **kwargs):result = kwargsif _metric:result[DEFAULT_METRIC] = _metric- trial = _runner.running_trial+ trial = getattr(_runner, "running_trial", None)+ if not trial:+ return Noneif _running_trial == trial:_training_iteration += 1else:@@ -102,11 +104,11 @@ def report(_metric=None, **kwargs):del result["config"][INCUMBENT_RESULT]for key, value in trial.config.items():result["config/" + key] = value- _runner.process_trial_result(_runner.running_trial, result)+ _runner.process_trial_result(trial, result)result["time_total_s"] = trial.last_update_time - trial.start_timeif _verbose > 2:logger.info(f"result: {result}")- if _runner.running_trial.is_finished():+ if trial.is_finished():return Noneelse:return Truediff --git a/flaml/version.py b/flaml/version.pyindex 50533e307..f5b77301f 100644--- a/flaml/version.py+++ b/flaml/version.py@@ -1 +1 @@-__version__ = "0.9.6"+__version__ = "0.9.7"diff --git a/test/.Docker/Dockerfile-cpu b/test/.Docker/Dockerfile-cpuindex 3c450b935..da2570cf4 100644--- a/test/.Docker/Dockerfile-cpu+++ b/test/.Docker/Dockerfile-cpu@@ -1,8 +1,8 @@-FROM python:3.7+FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04RUN pip install azureml-coreRUN pip install flaml[blendsearch,ray]-+RUN pip install ray-on-amlEXPOSE 8265EXPOSE 6379diff --git a/test/ray/distribute_automl.py b/test/ray/distribute_automl.pyindex c9d6c24be..14f15a0d0 100644--- a/test/ray/distribute_automl.py+++ b/test/ray/distribute_automl.py@@ -1,15 +1,17 @@+from ray_on_aml.core import Ray_On_AMLfrom flaml import AutoMLdef _test_ray_classification():from sklearn.datasets import make_classification- import ray- ray.init(address="auto")X, y = make_classification(1000, 10)automl = AutoML()automl.fit(X, y, time_budget=10, task="classification", n_concurrent_trials=2)if __name__ == "__main__":- _test_ray_classification()+ ray_on_aml = Ray_On_AML()+ ray = ray_on_aml.getRay()+ if ray:+ _test_ray_classification()diff --git a/test/ray/distribute_tune.py b/test/ray/distribute_tune.pyindex 7a737d16f..991082e8e 100644--- a/test/ray/distribute_tune.py+++ b/test/ray/distribute_tune.py@@ -1,4 +1,4 @@-import ray+from ray_on_aml.core import Ray_On_AMLimport lightgbm as lgbimport numpy as npfrom sklearn.datasets import load_breast_cancer@@ -7,11 +7,6 @@ from sklearn.model_selection import train_test_splitfrom flaml import tunefrom flaml.model import LGBMEstimator-X, y = load_breast_cancer(return_X_y=True)-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)-ray.init(address="auto")-X_train_ref = ray.put(X_train)-def train_breast_cancer(config):params = LGBMEstimator(**config).params@@ -24,25 +19,31 @@ def train_breast_cancer(config):if __name__ == "__main__":- flaml_lgbm_search_space = LGBMEstimator.search_space(X_train.shape)- config_search_space = {- hp: space["domain"] for hp, space in flaml_lgbm_search_space.items()- }- low_cost_partial_config = {- hp: space["low_cost_init_value"]- for hp, space in flaml_lgbm_search_space.items()- if "low_cost_init_value" in space- }+ ray_on_aml = Ray_On_AML()+ ray = ray_on_aml.getRay()+ if ray:+ X, y = load_breast_cancer(return_X_y=True)+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)+ X_train_ref = ray.put(X_train)+ flaml_lgbm_search_space = LGBMEstimator.search_space(X_train.shape)+ config_search_space = {+ hp: space["domain"] for hp, space in flaml_lgbm_search_space.items()+ }+ low_cost_partial_config = {+ hp: space["low_cost_init_value"]+ for hp, space in flaml_lgbm_search_space.items()+ if "low_cost_init_value" in space+ }- analysis = tune.run(- train_breast_cancer,- metric="mean_accuracy",- mode="max",- config=config_search_space,- num_samples=-1,- time_budget_s=60,- use_ray=True,- )+ analysis = tune.run(+ train_breast_cancer,+ metric="mean_accuracy",+ mode="max",+ config=config_search_space,+ num_samples=-1,+ time_budget_s=60,+ use_ray=True,+ )- # print("Best hyperparameters found were: ", analysis.best_config)- print("The best trial's result: ", analysis.best_trial.last_result)+ # print("Best hyperparameters found were: ", analysis.best_config)+ print("The best trial's result: ", analysis.best_trial.last_result)diff --git a/test/run_distribute_automl.py b/test/run_distribute_automl.pyindex fb6bc2336..8490a8c20 100644--- a/test/run_distribute_automl.py+++ b/test/run_distribute_automl.py@@ -1,5 +1,6 @@import timefrom azureml.core import Workspace, Experiment, ScriptRunConfig, Environment+from azureml.core.runconfig import RunConfiguration, DockerConfigurationws = Workspace.from_config()ray_environment_name = "aml-ray-cpu"@@ -18,20 +19,21 @@ while ray_cpu_build_details.status not in ["Succeeded", "Failed"]:)time.sleep(10)+command = ["python distribute_automl.py"]env = Environment.get(workspace=ws, name=ray_environment_name)compute_target = ws.compute_targets["cpucluster"]-command = ["python automl.py"]+aml_run_config = RunConfiguration(communicator="OpenMpi")+aml_run_config.target = compute_target+aml_run_config.docker = DockerConfiguration(use_docker=True)+aml_run_config.environment = env+aml_run_config.node_count = 2config = ScriptRunConfig(source_directory="ray/",command=command,- compute_target=compute_target,- environment=env,+ run_config=aml_run_config,)-config.run_config.node_count = 2-config.run_config.environment_variables["_AZUREML_CR_START_RAY"] = "true"-config.run_config.environment_variables["AZUREML_COMPUTE_USE_COMMON_RUNTIME"] = "true"-exp = Experiment(ws, "test-ray")+exp = Experiment(ws, "distribute-automl")run = exp.submit(config)print(run.get_portal_url()) # link to ml.azure.comrun.wait_for_completion(show_output=True)diff --git a/test/run_distribute_tune.py b/test/run_distribute_tune.pyindex 8f91d2592..34c35d9fd 100644--- a/test/run_distribute_tune.py+++ b/test/run_distribute_tune.py@@ -1,5 +1,6 @@import timefrom azureml.core import Workspace, Experiment, ScriptRunConfig, Environment+from azureml.core.runconfig import RunConfiguration, DockerConfigurationws = Workspace.from_config()ray_environment_name = "aml-ray-cpu"@@ -18,20 +19,21 @@ while ray_cpu_build_details.status not in ["Succeeded", "Failed"]:)time.sleep(10)+command = ["python distribute_tune.py"]env = Environment.get(workspace=ws, name=ray_environment_name)compute_target = ws.compute_targets["cpucluster"]-command = ["python distribute_tune.py"]+aml_run_config = RunConfiguration(communicator="OpenMpi")+aml_run_config.target = compute_target+aml_run_config.docker = DockerConfiguration(use_docker=True)+aml_run_config.environment = env+aml_run_config.node_count = 2config = ScriptRunConfig(source_directory="ray/",command=command,- compute_target=compute_target,- environment=env,+ run_config=aml_run_config,)-config.run_config.node_count = 2-config.run_config.environment_variables["_AZUREML_CR_START_RAY"] = "true"-config.run_config.environment_variables["AZUREML_COMPUTE_USE_COMMON_RUNTIME"] = "true"-exp = Experiment(ws, "test-ray")+exp = Experiment(ws, "distribute-tune")run = exp.submit(config)print(run.get_portal_url()) # link to ml.azure.comrun.wait_for_completion(show_output=True)diff --git a/website/docs/Examples/Integrate - AzureML.md b/website/docs/Examples/Integrate - AzureML.mdindex 4675970fd..fc085fa9a 100644--- a/website/docs/Examples/Integrate - AzureML.md+++ b/website/docs/Examples/Integrate - AzureML.md@@ -128,40 +128,35 @@ If the computer target "cpucluster" already exists, it will not be recreated.#### Run distributed AutoML job-Assuming you have an automl script like [ray/distribute_automl.py](https://github.com/microsoft/FLAML/blob/main/test/ray/distribute_automl.py). It uses `ray.init(address="auto")` to initialize the cluster, and uses `n_concurrent_trials=k` to inform `AutoML.fit()` to perform k concurrent trials in parallel.+Assuming you have an automl script like [ray/distribute_automl.py](https://github.com/microsoft/FLAML/blob/main/test/ray/distribute_automl.py). It uses `n_concurrent_trials=k` to inform `AutoML.fit()` to perform k concurrent trials in parallel.Submit an AzureML job as the following:```pythonfrom azureml.core import Workspace, Experiment, ScriptRunConfig, Environment+from azureml.core.runconfig import RunConfiguration, DockerConfigurationcommand = ["python distribute_automl.py"]-ray_environment_name = 'aml-ray-cpu'+ray_environment_name = "aml-ray-cpu"env = Environment.get(workspace=ws, name=ray_environment_name)+aml_run_config = RunConfiguration(communicator="OpenMpi")+aml_run_config.target = compute_target+aml_run_config.docker = DockerConfiguration(use_docker=True)+aml_run_config.environment = env+aml_run_config.node_count = 2config = ScriptRunConfig(- source_directory='ray/',+ source_directory="ray/",command=command,- compute_target=compute_target,- environment=env,+ run_config=aml_run_config,)-config.run_config.node_count = 2-config.run_config.environment_variables["_AZUREML_CR_START_RAY"] = "true"-config.run_config.environment_variables["AZUREML_COMPUTE_USE_COMMON_RUNTIME"] = "true"--exp = Experiment(ws, 'distribute-automl')+exp = Experiment(ws, "distribute-automl")run = exp.submit(config)print(run.get_portal_url()) # link to ml.azure.comrun.wait_for_completion(show_output=True)```-The line-`-config.run_config.environment_variables["_AZUREML_CR_START_RAY"] = "true"-`-tells AzureML to start ray on each node of the cluster. This ia a feature in preview and it is subject to change in future. It is applicable to dedicated VMs only.-#### Run distributed tune jobPrepare a script like [ray/distribute_tune.py](https://github.com/microsoft/FLAML/blob/main/test/ray/distribute_tune.py). Replace the command in the above eample with: