Skip to content

A programming framework for agentic AI

Gpu support for xgboost (#442)

Chi Wang6960a833eca9
diff --git a/flaml/automl.py b/flaml/automl.py
index 8fa0c2de6..7c8e2a0c7 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -1926,7 +1926,7 @@ class AutoML(BaseEstimator):
the searched learners, such as sample_weight. Include:
period: int | forecast horizon for 'ts_forecast' task.
gpu_per_trial: float, default = 0 | A float of the number of gpus per trial,
- only used by TransformersEstimator.
+ only used by TransformersEstimator and XGBoostSklearnEstimator.
"""
self._state._start_time_flag = self._start_time_flag = time.time()
@@ -2084,7 +2084,6 @@ class AutoML(BaseEstimator):
if _is_nlp_task(self._state.task):
self._state.fit_kwargs["metric"] = metric
self._state.fit_kwargs["use_ray"] = self._use_ray
- self._state.fit_kwargs["gpu_per_trial"] = gpu_per_trial
self._state.metric = metric
diff --git a/flaml/model.py b/flaml/model.py
index c0ecfddcf..27e9c3184 100644
--- a/flaml/model.py
+++ b/flaml/model.py
@@ -1,5 +1,5 @@
# !
-# * Copyright (c) Microsoft Corporation. All rights reserved.
+# * Copyright (c) FLAML authors. All rights reserved.
# * Licensed under the MIT License. See LICENSE file in the
# * project root for license information.
from contextlib import contextmanager
@@ -22,7 +22,6 @@ from . import tune
from .data import (
group_counts,
CLASSIFICATION,
- TS_FORECAST,
TS_FORECASTREGRESSION,
TS_TIMESTAMP_COL,
TS_VALUE_COL,
@@ -1185,6 +1184,9 @@ class XGBoostSklearnEstimator(SKLearnEstimator, LGBMEstimator):
def fit(self, X_train, y_train, budget=None, **kwargs):
if issparse(X_train):
self.params["tree_method"] = "auto"
+ if kwargs.get("gpu_per_trial"):
+ self.params["tree_method"] = "gpu_hist"
+ kwargs.pop("gpu_per_trial")
return super().fit(X_train, y_train, budget, **kwargs)
def _callbacks(self, start_time, deadline) -> List[Callable]:
diff --git a/test/nlp/run_gpu.py b/test/test_gpu.py
similarity index 66%
rename from test/nlp/run_gpu.py
rename to test/test_gpu.py
index 1574c7fc9..80bbcfc5f 100644
--- a/test/nlp/run_gpu.py
+++ b/test/test_gpu.py
@@ -4,6 +4,47 @@ import pickle
import shutil
+def test_xgboost():
+ from flaml import AutoML
+ from sklearn.datasets import make_moons
+ import scipy.sparse
+ import numpy as np
+ from xgboost.core import XGBoostError
+
+ try:
+ X_train = scipy.sparse.eye(900000)
+ y_train = np.random.randint(2, size=900000)
+ automl = AutoML()
+ automl.fit(
+ X_train,
+ y_train,
+ estimator_list=["xgb_limitdepth", "xgboost"],
+ time_budget=5,
+ gpu_per_trial=1,
+ )
+
+ train, label = make_moons(
+ n_samples=300000, shuffle=True, noise=0.3, random_state=None
+ )
+ automl = AutoML()
+ automl.fit(
+ train,
+ label,
+ estimator_list=["xgb_limitdepth", "xgboost"],
+ time_budget=5,
+ gpu_per_trial=1,
+ )
+ automl.fit(
+ train,
+ label,
+ estimator_list=["xgb_limitdepth", "xgboost"],
+ time_budget=5,
+ )
+ except XGBoostError:
+ # No visible GPU is found for XGBoost.
+ return
+
+
@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os")
def _test_hf_data():
from flaml import AutoML
diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py
index d923a8cc1..be2c011e7 100644
--- a/test/tune/test_tune.py
+++ b/test/tune/test_tune.py
@@ -6,14 +6,13 @@ import os
from sklearn.model_selection import train_test_split
import sklearn.metrics
import sklearn.datasets
+import xgboost as xgb
+import logging
try:
from ray.tune.integration.xgboost import TuneReportCheckpointCallback
except ImportError:
print("skip test_xgboost because ray tune cannot be imported.")
-import xgboost as xgb
-
-import logging
logger = logging.getLogger(__name__)
os.makedirs("logs", exist_ok=True)
@@ -66,8 +65,6 @@ def _test_xgboost(method="BlendSearch"):
time_budget_s = 60
for n_cpu in [2]:
start_time = time.time()
- ray.shutdown()
- ray.init(num_cpus=n_cpu, num_gpus=0)
# ray.init(address='auto')
if method == "BlendSearch":
analysis = tune.run(
@@ -169,7 +166,6 @@ def _test_xgboost(method="BlendSearch"):
scheduler=scheduler,
search_alg=algo,
)
- ray.shutdown()
# # Load the best model checkpoint
# import os
# best_bst = xgb.Booster()