Skip to content

A programming framework for agentic AI

align indent and add missing quotation (#555)

Chi Wangd402c633124f
diff --git a/flaml/automl.py b/flaml/automl.py
index e3ebaa40a..ca033e4e6 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -475,176 +475,176 @@ class AutoML(BaseEstimator):
def __init__(self, **settings):
"""Constructor.
- Many settings in fit() can be passed to the constructor too.
- If an argument in fit() is provided, it will override the setting passed to the constructor.
- If an argument in fit() is not provided but provided in the constructor, the value passed to the constructor will be used.
-
- Args:
- metric: A string of the metric name or a function,
- e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo',
- 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2',
- 'mape'. Default is 'auto'.
- If passing a customized metric function, the function needs to
- have the follwing signature:
-
- ```python
- def custom_metric(
- X_test, y_test, estimator, labels,
- X_train, y_train, weight_test=None, weight_train=None,
- config=None, groups_test=None, groups_train=None,
- ):
- return metric_to_minimize, metrics_to_log
- ```
- which returns a float number as the minimization objective,
- and a dictionary as the metrics to log. E.g.,
-
- ```python
- def custom_metric(
- X_val, y_val, estimator, labels,
- X_train, y_train, weight_val=None, weight_train=None,
- *args,
- ):
- from sklearn.metrics import log_loss
- import time
-
- start = time.time()
- y_pred = estimator.predict_proba(X_val)
- pred_time = (time.time() - start) / len(X_val)
- val_loss = log_loss(y_val, y_pred, labels=labels, sample_weight=weight_val)
- y_pred = estimator.predict_proba(X_train)
- train_loss = log_loss(y_train, y_pred, labels=labels, sample_weight=weight_train)
- alpha = 0.5
- return val_loss * (1 + alpha) - alpha * train_loss, {
- "val_loss": val_loss,
- "train_loss": train_loss,
- "pred_time": pred_time,
- }
- ```
- task: A string of the task type, e.g.,
- 'classification', 'regression', 'ts_forecast', 'rank',
- 'seq-classification', 'seq-regression', 'summarization'.
- n_jobs: An integer of the number of threads for training | default=-1.
- Use all available resources when n_jobs == -1.
- log_file_name: A string of the log file name | default="". To disable logging,
- set it to be an empty string "".
- estimator_list: A list of strings for estimator names, or 'auto'
- e.g., ```['lgbm', 'xgboost', 'xgb_limitdepth', 'catboost', 'rf', 'extra_tree']```
- time_budget: A float number of the time budget in seconds.
- Use -1 if no time limit.
- max_iter: An integer of the maximal number of iterations.
- sample: A boolean of whether to sample the training data during
- search.
+ Many settings in fit() can be passed to the constructor too.
+ If an argument in fit() is provided, it will override the setting passed to the constructor.
+ If an argument in fit() is not provided but provided in the constructor, the value passed to the constructor will be used.
+
+ Args:
+ metric: A string of the metric name or a function,
+ e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo',
+ 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2',
+ 'mape'. Default is 'auto'.
+ If passing a customized metric function, the function needs to
+ have the follwing signature:
+
+ ```python
+ def custom_metric(
+ X_test, y_test, estimator, labels,
+ X_train, y_train, weight_test=None, weight_train=None,
+ config=None, groups_test=None, groups_train=None,
+ ):
+ return metric_to_minimize, metrics_to_log
+ ```
+ which returns a float number as the minimization objective,
+ and a dictionary as the metrics to log. E.g.,
+
+ ```python
+ def custom_metric(
+ X_val, y_val, estimator, labels,
+ X_train, y_train, weight_val=None, weight_train=None,
+ *args,
+ ):
+ from sklearn.metrics import log_loss
+ import time
+
+ start = time.time()
+ y_pred = estimator.predict_proba(X_val)
+ pred_time = (time.time() - start) / len(X_val)
+ val_loss = log_loss(y_val, y_pred, labels=labels, sample_weight=weight_val)
+ y_pred = estimator.predict_proba(X_train)
+ train_loss = log_loss(y_train, y_pred, labels=labels, sample_weight=weight_train)
+ alpha = 0.5
+ return val_loss * (1 + alpha) - alpha * train_loss, {
+ "val_loss": val_loss,
+ "train_loss": train_loss,
+ "pred_time": pred_time,
+ }
+ ```
+ task: A string of the task type, e.g.,
+ 'classification', 'regression', 'ts_forecast', 'rank',
+ 'seq-classification', 'seq-regression', 'summarization'.
+ n_jobs: An integer of the number of threads for training | default=-1.
+ Use all available resources when n_jobs == -1.
+ log_file_name: A string of the log file name | default="". To disable logging,
+ set it to be an empty string "".
+ estimator_list: A list of strings for estimator names, or 'auto'
+ e.g., ```['lgbm', 'xgboost', 'xgb_limitdepth', 'catboost', 'rf', 'extra_tree']```
+ time_budget: A float number of the time budget in seconds.
+ Use -1 if no time limit.
+ max_iter: An integer of the maximal number of iterations.
+ sample: A boolean of whether to sample the training data during
+ search.
ensemble: boolean or dict | default=False. Whether to perform
ensemble after search. Can be a dict with keys 'passthrough'
and 'final_estimator' to specify the passthrough and
final_estimator in the stacker. The dict can also contain
'n_jobs' as the key to specify the number of jobs for the stacker.
- eval_method: A string of resampling strategy, one of
- ['auto', 'cv', 'holdout'].
- split_ratio: A float of the valiation data percentage for holdout.
- n_splits: An integer of the number of folds for cross - validation.
- log_type: A string of the log type, one of
- ['better', 'all'].
- 'better' only logs configs with better loss than previos iters
- 'all' logs all the tried configs.
- model_history: A boolean of whether to keep the best
- model per estimator. Make sure memory is large enough if setting to True.
- log_training_metric: A boolean of whether to log the training
- metric for each model.
- mem_thres: A float of the memory size constraint in bytes.
- pred_time_limit: A float of the prediction latency constraint in seconds.
- It refers to the average prediction time per row in validation data.
- train_time_limit: A float of the training time constraint in seconds.
- verbose: int, default=3 | Controls the verbosity, higher means more
- messages.
- retrain_full: bool or str, default=True | whether to retrain the
- selected model on the full training data when using holdout.
- True - retrain only after search finishes; False - no retraining;
- 'budget' - do best effort to retrain without violating the time
- budget.
- split_type: str or splitter object, default="auto" | the data split type.
- * A valid splitter object is an instance of a derived class of scikit-learn
- [KFold](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.KFold.html#sklearn.model_selection.KFold)
- and have ``split`` and ``get_n_splits`` methods with the same signatures.
- Set eval_method to "cv" to use the splitter object.
- * Valid str options depend on different tasks.
- For classification tasks, valid choices are
- ["auto", 'stratified', 'uniform', 'time', 'group']. "auto" -> stratified.
- For regression tasks, valid choices are ["auto", 'uniform', 'time'].
- "auto" -> uniform.
- For ts_forecast tasks, must be "auto" or 'time'.
- For ranking task, must be "auto" or 'group'.
- hpo_method: str, default="auto" | The hyperparameter
- optimization method. By default, CFO is used for sequential
- search and BlendSearch is used for parallel search.
- No need to set when using flaml's default search space or using
- a simple customized search space. When set to 'bs', BlendSearch
- is used. BlendSearch can be tried when the search space is
- complex, for example, containing multiple disjoint, discontinuous
- subspaces. When set to 'random', random search is used.
- starting_points: A dictionary or a str to specify the starting hyperparameter
- config for the estimators | default="static".
- If str:
- - if "data", use data-dependent defaults;
- - if "data:path" use data-dependent defaults which are stored at path;
- - if "static", use data-independent defaults.
- If dict, keys are the name of the estimators, and values are the starting
- hyperparamter configurations for the corresponding estimators.
- The value can be a single hyperparamter configuration dict or a list
- of hyperparamter configuration dicts.
- In the following code example, we get starting_points from the
- `automl` object and use them in the `new_automl` object.
- e.g.,
-
- ```python
- from flaml import AutoML
- automl = AutoML()
- X_train, y_train = load_iris(return_X_y=True)
- automl.fit(X_train, y_train)
- starting_points = automl.best_config_per_estimator
-
- new_automl = AutoML()
- new_automl.fit(X_train, y_train, starting_points=starting_points)
- ```
-
- seed: int or None, default=None | The random seed for hpo.
- n_concurrent_trials: [Experimental] int, default=1 | The number of
- concurrent trials. When n_concurrent_trials > 1, flaml performes
- [parallel tuning](https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#parallel-tuning)
- and installation of ray is required: `pip install flaml[ray]`.
- keep_search_state: boolean, default=False | Whether to keep data needed
- for model search after fit(). By default the state is deleted for
- space saving.
- early_stop: boolean, default=False | Whether to stop early if the
- search is considered to converge.
- append_log: boolean, default=False | Whetehr to directly append the log
- records to the input log file if it exists.
- auto_augment: boolean, default=True | Whether to automatically
- augment rare classes.
- min_sample_size: int, default=MIN_SAMPLE_TRAIN | the minimal sample
- size when sample=True.
- use_ray: boolean, default=False | Whether to use ray to run the training
- in separate processes. This can be used to prevent OOM for large
- datasets, but will incur more overhead in time. Only use it if
- you run into OOM failures.
- metric_constraints: list, default=[] | The list of metric constraints.
- Each element in this list is a 3-tuple, which shall be expressed
- in the following format: the first element of the 3-tuple is the name of the
- metric, the second element is the inequality sign chosen from ">=" and "<=",
- and the third element is the constraint value. E.g., `('val_loss', '<=', 0.1)`.
- Note that all the metric names in metric_constraints need to be reported via
- the metrics_to_log dictionary returned by a customized metric function.
- The customized metric function shall be provided via the `metric` key word
- argument of the fit() function or the automl constructor.
- Find an example in the 4th constraint type in this [doc](https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#constraint).
- If `pred_time_limit` is provided as one of keyword arguments to fit() function or
- the automl constructor, flaml will automatically (and under the hood)
- add it as an additional element in the metric_constraints. Essentially 'pred_time_limit'
- specifies a constraint about the prediction latency constraint in seconds.
- custom_hp: dict, default=None | The custom search space specified by user
- Each key is the estimator name, each value is a dict of the custom search space for that estimator. Notice the
- domain of the custom search space can either be a value of a sample.Domain object.
- e.g.,
+ eval_method: A string of resampling strategy, one of
+ ['auto', 'cv', 'holdout'].
+ split_ratio: A float of the valiation data percentage for holdout.
+ n_splits: An integer of the number of folds for cross - validation.
+ log_type: A string of the log type, one of
+ ['better', 'all'].
+ 'better' only logs configs with better loss than previos iters
+ 'all' logs all the tried configs.
+ model_history: A boolean of whether to keep the best
+ model per estimator. Make sure memory is large enough if setting to True.
+ log_training_metric: A boolean of whether to log the training
+ metric for each model.
+ mem_thres: A float of the memory size constraint in bytes.
+ pred_time_limit: A float of the prediction latency constraint in seconds.
+ It refers to the average prediction time per row in validation data.
+ train_time_limit: A float of the training time constraint in seconds.
+ verbose: int, default=3 | Controls the verbosity, higher means more
+ messages.
+ retrain_full: bool or str, default=True | whether to retrain the
+ selected model on the full training data when using holdout.
+ True - retrain only after search finishes; False - no retraining;
+ 'budget' - do best effort to retrain without violating the time
+ budget.
+ split_type: str or splitter object, default="auto" | the data split type.
+ * A valid splitter object is an instance of a derived class of scikit-learn
+ [KFold](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.KFold.html#sklearn.model_selection.KFold)
+ and have ``split`` and ``get_n_splits`` methods with the same signatures.
+ Set eval_method to "cv" to use the splitter object.
+ * Valid str options depend on different tasks.
+ For classification tasks, valid choices are
+ ["auto", 'stratified', 'uniform', 'time', 'group']. "auto" -> stratified.
+ For regression tasks, valid choices are ["auto", 'uniform', 'time'].
+ "auto" -> uniform.
+ For ts_forecast tasks, must be "auto" or 'time'.
+ For ranking task, must be "auto" or 'group'.
+ hpo_method: str, default="auto" | The hyperparameter
+ optimization method. By default, CFO is used for sequential
+ search and BlendSearch is used for parallel search.
+ No need to set when using flaml's default search space or using
+ a simple customized search space. When set to 'bs', BlendSearch
+ is used. BlendSearch can be tried when the search space is
+ complex, for example, containing multiple disjoint, discontinuous
+ subspaces. When set to 'random', random search is used.
+ starting_points: A dictionary or a str to specify the starting hyperparameter
+ config for the estimators | default="static".
+ If str:
+ - if "data", use data-dependent defaults;
+ - if "data:path" use data-dependent defaults which are stored at path;
+ - if "static", use data-independent defaults.
+ If dict, keys are the name of the estimators, and values are the starting
+ hyperparamter configurations for the corresponding estimators.
+ The value can be a single hyperparamter configuration dict or a list
+ of hyperparamter configuration dicts.
+ In the following code example, we get starting_points from the
+ `automl` object and use them in the `new_automl` object.
+ e.g.,
+
+ ```python
+ from flaml import AutoML
+ automl = AutoML()
+ X_train, y_train = load_iris(return_X_y=True)
+ automl.fit(X_train, y_train)
+ starting_points = automl.best_config_per_estimator
+
+ new_automl = AutoML()
+ new_automl.fit(X_train, y_train, starting_points=starting_points)
+ ```
+
+ seed: int or None, default=None | The random seed for hpo.
+ n_concurrent_trials: [Experimental] int, default=1 | The number of
+ concurrent trials. When n_concurrent_trials > 1, flaml performes
+ [parallel tuning](https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#parallel-tuning)
+ and installation of ray is required: `pip install flaml[ray]`.
+ keep_search_state: boolean, default=False | Whether to keep data needed
+ for model search after fit(). By default the state is deleted for
+ space saving.
+ early_stop: boolean, default=False | Whether to stop early if the
+ search is considered to converge.
+ append_log: boolean, default=False | Whetehr to directly append the log
+ records to the input log file if it exists.
+ auto_augment: boolean, default=True | Whether to automatically
+ augment rare classes.
+ min_sample_size: int, default=MIN_SAMPLE_TRAIN | the minimal sample
+ size when sample=True.
+ use_ray: boolean, default=False | Whether to use ray to run the training
+ in separate processes. This can be used to prevent OOM for large
+ datasets, but will incur more overhead in time. Only use it if
+ you run into OOM failures.
+ metric_constraints: list, default=[] | The list of metric constraints.
+ Each element in this list is a 3-tuple, which shall be expressed
+ in the following format: the first element of the 3-tuple is the name of the
+ metric, the second element is the inequality sign chosen from ">=" and "<=",
+ and the third element is the constraint value. E.g., `('val_loss', '<=', 0.1)`.
+ Note that all the metric names in metric_constraints need to be reported via
+ the metrics_to_log dictionary returned by a customized metric function.
+ The customized metric function shall be provided via the `metric` key word
+ argument of the fit() function or the automl constructor.
+ Find an example in the 4th constraint type in this [doc](https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#constraint).
+ If `pred_time_limit` is provided as one of keyword arguments to fit() function or
+ the automl constructor, flaml will automatically (and under the hood)
+ add it as an additional element in the metric_constraints. Essentially 'pred_time_limit'
+ specifies a constraint about the prediction latency constraint in seconds.
+ custom_hp: dict, default=None | The custom search space specified by user
+ Each key is the estimator name, each value is a dict of the custom search space for that estimator. Notice the
+ domain of the custom search space can either be a value of a sample.Domain object.
+ e.g.,
```python
custom_hp = {
@@ -657,19 +657,19 @@ class AutoML(BaseEstimator):
}
}
}
- ```
- fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name.
- e.g.,
-
- ```python
- fit_kwargs_by_estimator = {
- "transformer": {
- "output_dir": "test/data/output/",
- "ckpt_per_epoch": 1,
- "fp16": False,
- }
- }
- ```
+ ```
+ fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name.
+ e.g.,
+
+ ```python
+ fit_kwargs_by_estimator = {
+ "transformer": {
+ "output_dir": "test/data/output/",
+ "ckpt_per_epoch": 1,
+ "fp16": False,
+ }
+ }
+ ```
"""
self._track_iter = 0
@@ -1558,6 +1558,7 @@ class AutoML(BaseEstimator):
}
}
}
+ ```
fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name.
e.g.,