A programming framework for agentic AI
variable name (#187)
Chi Wang339eb80f4404
diff --git a/flaml/automl.py b/flaml/automl.pyindex a118891be..f330d9b8b 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -366,7 +366,7 @@ class AutoML:@propertydef classes_(self):'''A list of n_classes elements for class labels.'''- attr = getattr(self, "label_transformer", None)+ attr = getattr(self, "_label_transformer", None)if attr:return attr.classes_.tolist()attr = getattr(self, "_trained_estimator", None)diff --git a/flaml/nlp/autotransformers.py b/flaml/nlp/autotransformers.pyindex 9bf521b5e..096f2f986 100644--- a/flaml/nlp/autotransformers.py+++ b/flaml/nlp/autotransformers.py@@ -87,6 +87,7 @@ class AutoTransformers:@staticmethoddef _get_split_name(data_raw, fold_name=None):+ # TODO coverageif fold_name:return fold_namefold_keys = data_raw.keys()@@ -280,6 +281,7 @@ class AutoTransformers:model_config = _set_model_config()if is_pretrained_model_in_classification_head_list():+ # TODO coverageif self._num_labels != num_labels_old:this_model = get_this_model()model_config.num_labels = self._num_labels@@ -295,6 +297,7 @@ class AutoTransformers:this_model.resize_token_embeddings(len(self._tokenizer))return this_modelelif this_task == "regression":+ # TODO add testmodel_config_num_labels = 1model_config = _set_model_config()this_model = get_this_model()@@ -304,6 +307,7 @@ class AutoTransformers:data_name = JobID.dataset_list_to_str(self.jobid_config.dat)if data_name in ("glue", "super_glue"):metric = datasets.load.load_metric(data_name, self.jobid_config.subdat)+ # TODO deleteelif data_name in ("squad", "squad_v2"):metric = datasets.load.load_metric(data_name)else:@@ -312,6 +316,7 @@ class AutoTransformers:def _compute_metrics_by_dataset_name(self,eval_pred):+ # TODO coveragepredictions, labels = eval_predpredictions = np.squeeze(predictions) \if self.task_name == "regression" else np.argmax(predictions, axis=1)@@ -321,6 +326,7 @@ class AutoTransformers:def _compute_checkpoint_freq(self,num_train_epochs,batch_size):+ # TODO coverageif "gpu" in self._resources_per_trial:ckpt_step_freq = int(min(num_train_epochs, 1) * len(self.train_dataset) / batch_size/ self._resources_per_trial["gpu"] / self.ckpt_per_epoch) + 1@@ -544,6 +550,7 @@ class AutoTransformers:_fp16=True,**custom_hpo_args):+ # TODO remove?from transformers.trainer_utils import HPSearchBackend'''Fine tuning the huggingface using HF's API Transformers.hyperparameter_search (for comparitive purpose).@@ -657,6 +664,7 @@ class AutoTransformers:return validation_metricdef _set_transformers_verbosity(self, transformers_verbose):+ # TODO coverageif transformers_verbose == transformers.logging.ERROR:transformers.logging.set_verbosity_error()elif transformers_verbose == transformers.logging.WARNING:diff --git a/flaml/nlp/dataset/dataprocess_auto.py b/flaml/nlp/dataset/dataprocess_auto.pyindex 058610963..09e421891 100644--- a/flaml/nlp/dataset/dataprocess_auto.py+++ b/flaml/nlp/dataset/dataprocess_auto.py@@ -77,6 +77,7 @@ def tokenize_superglue_wic(this_example,try:padding_direction = this_tokenizer.padding_sideif padding_direction == "left":+ # TODO coveragepadding_id = input_ids_sepp[0]while input_ids_sepp[ptr_sepp] == padding_id:ptr_sepp += 1@@ -103,6 +104,7 @@ def tokenize_superglue_wic(this_example,which_sepp += 1ptr_sepp += 1else:+ # TODO coverageptr_sepp += 1"""max_word_span is the maximum tokens of the word@@ -131,6 +133,7 @@ def tokenize_glue(this_example,if len(sentence_keys) > 1:sentence1_key, sentence2_key = sentence_keys[0], sentence_keys[1]else:+ # TODO coveragesentence1_key = sentence_keys[0]sentence2_key = Nonediff --git a/flaml/nlp/dataset/metric_auto.py b/flaml/nlp/dataset/metric_auto.pyindex c0210c32b..4abda815f 100644--- a/flaml/nlp/dataset/metric_auto.py+++ b/flaml/nlp/dataset/metric_auto.py@@ -64,6 +64,7 @@ def get_default_and_alternative_metric(dataset_name_list: typing.List,return default_metric, default_mode, all_metrics, all_modeelse:+ # TODO coverageassert isinstance(eval_name_mapping, list), "dataset_name and subdataset_name not correctly specified"default_metric, default_mode = eval_name_mapping[0]diff --git a/flaml/nlp/dataset/submission_auto.py b/flaml/nlp/dataset/submission_auto.pyindex 2c63337b0..bf49fd3d0 100644--- a/flaml/nlp/dataset/submission_auto.py+++ b/flaml/nlp/dataset/submission_auto.py@@ -59,6 +59,7 @@ def output_prediction_glue(output_path, zip_file_name, predictions, train_data,if subdataset_name != "mnli":is_match = subdataset_name == each_subdataset_nameelse:+ # TODO coverageif dev_name == "validation_matched":is_match = each_file == "MNLI-m.tsv"else:@@ -68,11 +69,13 @@ def output_prediction_glue(output_path, zip_file_name, predictions, train_data,writer.write("index\tprediction\n")for index, item in enumerate(predictions):if subdataset_name == "stsb":+ # TODO coverageif item > 5.0:item = 5.0writer.write(f"{index}\t{item:3.3f}\n")else:if subdataset_name in ("rte", "qnli", "mnli"):+ # TODO coverageitem = label_list[item]writer.write(f"{index}\t{item}\n")else:@@ -80,6 +83,7 @@ def output_prediction_glue(output_path, zip_file_name, predictions, train_data,item = int(item)writer.write(f"{index}\t{item}\n")else:+ # TODO coveragewriter.write(f"{index}\t{item:3.3f}\n")shutil.make_archive(os.path.join(output_path, zip_file_name), 'zip', output_dir)diff --git a/flaml/nlp/dataset/task_auto.py b/flaml/nlp/dataset/task_auto.pyindex bdc3335f6..4cdfa03c4 100644--- a/flaml/nlp/dataset/task_auto.py+++ b/flaml/nlp/dataset/task_auto.py@@ -42,6 +42,7 @@ def get_default_task(dataset_name_list: list, subdataset_name=None):"dataset_name and subdataset_name not correctly specified"default_task = eval_name_mapping[subdataset_name]else:+ # TODO coverageassert isinstance(eval_name_mapping, list), "dataset_name and subdataset_name not correctly specified"default_task = eval_name_mappingreturn default_taskdiff --git a/flaml/nlp/hpo/hpo_searchspace.py b/flaml/nlp/hpo/hpo_searchspace.pyindex 8b8d107e8..7e0b440a8 100644--- a/flaml/nlp/hpo/hpo_searchspace.py+++ b/flaml/nlp/hpo/hpo_searchspace.py@@ -33,8 +33,10 @@ def bounded_gridunion(model_type=None,if "u" in custom_hpo_args["bound"][each_key]:upper = custom_hpo_args["bound"][each_key]["u"]else:+ # TODO coverageupper = 100000if "l" in custom_hpo_args["bound"][each_key]:+ # TODO coveragelower = custom_hpo_args["bound"][each_key]["l"]else:lower = -100000@@ -42,6 +44,7 @@ def bounded_gridunion(model_type=None,upper_id = len(original_space)for x in range(len(original_space)):if original_space[x] > upper:+ # TODO coverageupper_id = xbreaklower_id = 0@@ -121,6 +124,7 @@ def hpo_space_generic_grid(model_type=None,subdataset_name=None,algo_mode=None,**custom_hpo_args):+ # TODO coverageoutput_config = {"learning_rate": [1e-5, 2e-5, 3e-5, 4e-5, 5e-5, 1e-4, 1.5e-4],"num_train_epochs": [3, 10],@@ -137,6 +141,7 @@ def hpo_space_small(model_type=None,subdataset_name=None,algo_mode=None,**custom_hpo_args):+ # TODO coverageconfig_json = AutoGridSearchSpace.from_model_and_dataset_name(model_type, model_size_type, dataset_name_list, subdataset_name, "hpo")output_config = {}diff --git a/flaml/nlp/hpo/searchalgo_auto.py b/flaml/nlp/hpo/searchalgo_auto.pyindex 1895c880d..4a180cfac 100644--- a/flaml/nlp/hpo/searchalgo_auto.py+++ b/flaml/nlp/hpo/searchalgo_auto.py@@ -70,9 +70,11 @@ class AutoSearchAlgorithm:assert hpo_search_space, "hpo_search_space needs to be specified for calling AutoSearchAlgorithm.from_method_name"if not search_algo_name:+ # TODO coveragesearch_algo_name = "grid"if search_algo_name in SEARCH_ALGO_MAPPING.keys():if SEARCH_ALGO_MAPPING[search_algo_name] is None:+ # TODO coveragereturn None"""filtering the customized args for hpo from custom_hpo_args, keep those@@ -91,6 +93,7 @@ class AutoSearchAlgorithm:: max(hpo_search_space["per_device_train_batch_size"].categories)},"""if search_algo_args_mode == "dft":+ # TODO coveragethis_search_algo_kwargs = DEFAULT_SEARCH_ALGO_ARGS_MAPPING[search_algo_name]("dft",metric_name,@@ -121,6 +124,7 @@ class AutoSearchAlgorithm:@staticmethoddef grid2list(grid_config):+ # TODO coveragekey_val_list = [[(key, each_val) for each_val in val_list['grid_search']]for (key, val_list) in grid_config.items()]config_list = [dict(x) for x in itertools.product(*key_val_list)]@@ -132,6 +136,7 @@ def get_search_algo_args_optuna(search_args_mode,metric_mode_name,hpo_search_space=None,**custom_hpo_args):+ # TODO coveragereturn {}@@ -145,6 +150,7 @@ def default_search_algo_args_bs(search_args_mode,isinstance(hpo_search_space["num_train_epochs"], ray.tune.sample.Categorical):min_epoch = min(hpo_search_space["num_train_epochs"].categories)else:+ # TODO coverageassert isinstance(hpo_search_space["num_train_epochs"], ray.tune.sample.Float)min_epoch = hpo_search_space["num_train_epochs"].lowerdefault_search_algo_args = {@@ -166,6 +172,7 @@ def default_search_algo_args_grid_search(search_args_mode,metric_mode_name,hpo_search_space=None,**custom_hpo_args):+ # TODO coveragereturn {}@@ -174,6 +181,7 @@ def default_search_algo_args_random_search(search_args_mode,metric_mode_name,hpo_search_space=None,**custom_hpo_args):+ # TODO coveragereturn {}diff --git a/flaml/nlp/huggingface/trainer.py b/flaml/nlp/huggingface/trainer.pyindex 53ac6a323..d1560968a 100644--- a/flaml/nlp/huggingface/trainer.py+++ b/flaml/nlp/huggingface/trainer.py@@ -17,6 +17,7 @@ class TrainerForAutoTransformers(TFTrainer):eval_dataset:the dataset to be evaluated"""+ # TODO coveragefrom ray import tuneeval_dataloader = self.get_eval_dataloader(eval_dataset)@@ -38,6 +39,7 @@ class TrainerForAutoTransformers(TFTrainer):Overriding transformers.Trainer.save_state. It is only through savingthe states can best_trial.get_best_checkpoint return a non-empty value."""+ # TODO coverageimport torchfrom transformers.trainer_utils import PREFIX_CHECKPOINT_DIRfrom ray import tune@@ -80,6 +82,7 @@ class TrainerForAutoTransformers(TFTrainer):device_count=None):if max_steps:return int(warmup_ratio * max_steps)+ # TODO coveragemax_steps = TrainerForAutoTransformers.convert_num_train_epochs_to_max_steps(num_train_epochs,num_train_examples,diff --git a/flaml/nlp/result_analysis/azure_utils.py b/flaml/nlp/result_analysis/azure_utils.pyindex 850cf3cfd..7aee85ad9 100644--- a/flaml/nlp/result_analysis/azure_utils.py+++ b/flaml/nlp/result_analysis/azure_utils.py@@ -360,6 +360,7 @@ class JobID:print("console_args has no attribute {}, continue".format(each_key))continueif self.mod == "grid":+ # TODO coverageself.alg = "grid"@@ -447,6 +448,7 @@ class AzureUtils:if autohf is not None:self.jobid = autohf.jobid_configelse:+ # TODO coverageassert jobid_config is not None, "jobid_config must be passed either through autohf.jobid_config" \" or jobid_config"self.jobid = jobid_config@@ -527,6 +529,7 @@ class AzureUtils:print("Your output will not be synced to azure because azure-blob-storage is not installed")def download_azure_blob(self, blobname):+ # TODO coverageblob_client = self._init_blob_client(blobname)if blob_client:pathlib.Path(re.search("(?P<parent_path>^.*)/[^/]+$", blobname).group("parent_path")).mkdir(@@ -549,6 +552,7 @@ class AzureUtils:metric_score = each_trial.metric_analysis["eval_" + analysis.default_metric]time_stamp = each_trial.metric_analysis['timestamp']except KeyError:+ # TODO coverageprint("KeyError, {} does not contain the key {} or {}".format("each_trial.metric_analysis","eval_" + analysis.default_metric,"timestamp"))@@ -608,6 +612,7 @@ class AzureUtils:"""azure_save_file_name = local_json_file.split("/")[-1][:-5]if self.data_root_dir is None:+ # TODO coveragefrom ..utils import load_dft_argsconsole_args = load_dft_args()output_dir = getattr(console_args, "data_root_dir")@@ -622,6 +627,7 @@ class AzureUtils:@staticmethoddef is_after_earliest_time(this_blob, earliest_time: Tuple[int, int, int]):+ # TODO coverageimport pytzutc = pytz.UTCif this_blob.last_modified >= utc.localize(datetime(earliest_time[0], earliest_time[1], earliest_time[2])):@@ -639,6 +645,7 @@ class AzureUtils:container_client = self._init_azure_clients()if container_client:for each_blob in container_client.list_blobs():+ # TODO coverageif each_blob.name.startswith(root_log_path):each_jobconfig = JobID.convert_blobname_to_jobid(each_blob.name)is_append = False@@ -701,6 +708,7 @@ class AzureUtils:"""matched_config_score_lists = []for (each_jobconfig, each_blob) in matched_blob_list:+ # TODO coverageself.download_azure_blob(each_blob.name)data_json = json.load(open(each_blob.name, "r"))each_config_and_score_list = ConfigScoreList(diff --git a/flaml/nlp/result_analysis/wandb_utils.py b/flaml/nlp/result_analysis/wandb_utils.pyindex 502f17d5f..9c3c4adb9 100644--- a/flaml/nlp/result_analysis/wandb_utils.py+++ b/flaml/nlp/result_analysis/wandb_utils.py@@ -35,6 +35,7 @@ class WandbUtils:os.environ["WANDB_API_KEY"] = wandb_keyos.environ["WANDB_MODE"] = "online"else:+ # TODO coverageos.environ["WANDB_MODE"] = "disabled"self.jobid_config = jobid_config@@ -53,6 +54,7 @@ class WandbUtils:return ""def set_wandb_per_trial(self):+ # TODO coverageprint("before wandb.init\n\n\n")try:import wandbdiff --git a/flaml/version.py b/flaml/version.pyindex 43c4ab005..22049ab2c 100644--- a/flaml/version.py+++ b/flaml/version.py@@ -1 +1 @@-__version__ = "0.6.1"+__version__ = "0.6.2"