A programming framework for agentic AI
fix the post-processing bug in NER (#534)
Xueqing Liu2a8decdc50bb
diff --git a/flaml/automl.py b/flaml/automl.pyindex 90d9ebf94..cc455237b 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -1569,7 +1569,10 @@ class AutoML(BaseEstimator):```**fit_kwargs: Other key word arguments to pass to fit() function of- the searched learners, such as sample_weight.+ the searched learners, such as sample_weight. Include:+ period: int | forecast horizon for ts_forecast tasks.+ gpu_per_trial: float, default = 0 | A float of the number of gpus per trial,+ only used by TransformersEstimator and XGBoostSklearnEstimator."""task = task or self._settings.get("task")eval_method = eval_method or self._settings.get("eval_method")@@ -2198,6 +2201,8 @@ class AutoML(BaseEstimator):}}fit_kwargs_by_estimator: dict, default=None | The user specified keywords arguments, grouped by estimator name.+ For TransformersEstimator, available fit_kwargs can be found from+ [flaml/nlp/training_args.py:TrainingArgumentsForAuto](https://microsoft.github.io/FLAML/docs/reference/nlp/huggingface/training_args).e.g.,```pythondiff --git a/flaml/ml.py b/flaml/ml.pyindex 51b6fb600..092a02565 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -165,7 +165,7 @@ def metric_loss_score(import datasetsdatasets_metric_name = huggingface_submetric_to_metric.get(- metric_name, metric_name+ metric_name, metric_name.split(":")[0])metric = datasets.load_metric(datasets_metric_name)metric_mode = huggingface_metric_to_mode[datasets_metric_name]@@ -174,17 +174,30 @@ def metric_loss_score(score = metric.compute(predictions=y_predict, references=y_true)[metric_name].mid.fmeasure- elif metric_name == "seqeval":- y_true = [- [x for x in each_y_true if x != -100] for each_y_true in y_true+ elif metric_name.startswith("seqeval"):++ zip_pred_true = [+ [(p, lb) for (p, lb) in zip(prediction, label) if lb != -100]+ for (prediction, label) in zip(y_predict, y_true)]y_pred = [- y_predict[each_idx][: len(y_true[each_idx])]- for each_idx in range(len(y_predict))+ [labels[p] for (p, l) in each_list]+ for each_list in zip_pred_true+ ] # To compute precision and recall, y_pred and y_true must be converted to string labels+ # (B-PER, I-PER, etc.), so that the category-based precision/recall (i.e., PER, LOC, etc.) scores can be computed+ y_true = [+ [labels[l] for (p, l) in each_list]+ for each_list in zip_pred_true]++ metric_submetric_names = metric_name.split(":")+score = metric.compute(predictions=y_pred, references=y_true)[- "overall_accuracy"+ metric_submetric_names[1]+ if len(metric_submetric_names) > 1+ else "overall_accuracy"]+else:score = metric.compute(predictions=y_predict, references=y_true)[metric_name@@ -454,7 +467,9 @@ def evaluate_model_CV(if task in CLASSIFICATION:labels = np.unique(y_train_all)else:- labels = None+ labels = fit_kwargs.get(+ "label_list"+ ) # pass the label list on to compute the evaluation metricgroups = Noneshuffle = False if task in TS_FORECAST else Trueif isinstance(kf, RepeatedStratifiedKFold):@@ -586,6 +601,9 @@ def compute_estimator(groups_val,eval_metric,task,+ labels=fit_kwargs.get(+ "label_list"+ ), # pass the label list on to compute the evaluation metricbudget=budget,log_training_metric=log_training_metric,fit_kwargs=fit_kwargs,diff --git a/flaml/model.py b/flaml/model.pyindex ec3ecad0f..0ecac416f 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -365,9 +365,9 @@ class TransformersEstimator(BaseEstimator):self._TrainingArguments = TrainingArguments@staticmethod- def _join(X_train, y_train):+ def _join(X_train, y_train, task):y_train = DataFrame(y_train, index=X_train.index)- y_train.columns = ["label"]+ y_train.columns = ["label"] if task != TOKENCLASSIFICATION else ["labels"]train_df = X_train.join(y_train)return train_df@@ -380,7 +380,7 @@ class TransformersEstimator(BaseEstimator):},"num_train_epochs": {"domain": tune.loguniform(lower=0.1, upper=10.0),- "init_value": 1,+ "init_value": 3.0, # to be consistent with roberta},"per_device_train_batch_size": {"domain": tune.choice([4, 8, 16, 32]),@@ -511,7 +511,7 @@ class TransformersEstimator(BaseEstimator):processed_X, processed_y = self._preprocess(X=X, y=y, **self._kwargs)processed_dataset = Dataset.from_pandas(- TransformersEstimator._join(processed_X, processed_y)+ TransformersEstimator._join(processed_X, processed_y, self._task))return processed_dataset, processed_X, processed_y@@ -547,14 +547,14 @@ class TransformersEstimator(BaseEstimator):@propertydef data_collator(self):- from .nlp.huggingface.data_collator import DataCollatorForAuto+ from .nlp.huggingface.data_collator import task_to_datacollator_classreturn (- DataCollatorForAuto(+ task_to_datacollator_class[self._task](tokenizer=self.tokenizer,- pad_to_multiple_of=8 if self._training_args.fp16 else None,+ pad_to_multiple_of=8, # if self._training_args.fp16 else None,)- if self._task == MULTICHOICECLASSIFICATION+ if self._task in (MULTICHOICECLASSIFICATION, TOKENCLASSIFICATION)else None)@@ -750,7 +750,10 @@ class TransformersEstimator(BaseEstimator):)metric_dict = {"automl_metric": metric_loss_score(- metric_name=self._metric, y_predict=predictions, y_true=labels+ metric_name=self._metric,+ y_predict=predictions,+ y_true=labels,+ labels=self._training_args.label_list,)}else:diff --git a/flaml/nlp/huggingface/data_collator.py b/flaml/nlp/huggingface/data_collator.pyindex bc8284c6a..1203a536c 100644--- a/flaml/nlp/huggingface/data_collator.py+++ b/flaml/nlp/huggingface/data_collator.py@@ -1,9 +1,15 @@from dataclasses import dataclass-from transformers.data.data_collator import DataCollatorWithPadding+from transformers.data.data_collator import (+ DataCollatorWithPadding,+ DataCollatorForTokenClassification,+)+from collections import OrderedDict++from flaml.data import TOKENCLASSIFICATION, MULTICHOICECLASSIFICATION@dataclass-class DataCollatorForAuto(DataCollatorWithPadding):+class DataCollatorForMultipleChoiceClassification(DataCollatorWithPadding):def __call__(self, features):from itertools import chainimport torch@@ -22,7 +28,9 @@ class DataCollatorForAuto(DataCollatorWithPadding):for feature in features]flattened_features = list(chain(*flattened_features))- batch = super(DataCollatorForAuto, self).__call__(flattened_features)+ batch = super(DataCollatorForMultipleChoiceClassification, self).__call__(+ flattened_features+ )# Un-flattenbatch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()}# Add back labels@@ -31,18 +39,9 @@ class DataCollatorForAuto(DataCollatorWithPadding):return batch-class DataCollatorForPredict(DataCollatorWithPadding):- def __call__(self, features):- from itertools import chain-- batch_size = len(features)- num_choices = len(features[0]["input_ids"])- flattened_features = [- [{k: v[i] for k, v in feature.items()} for i in range(num_choices)]- for feature in features- ]- flattened_features = list(chain(*flattened_features))- batch = super(DataCollatorForPredict, self).__call__(flattened_features)- # Un-flatten- batch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()}- return batch+task_to_datacollator_class = OrderedDict(+ [+ (TOKENCLASSIFICATION, DataCollatorForTokenClassification),+ (MULTICHOICECLASSIFICATION, DataCollatorForMultipleChoiceClassification),+ ]+)diff --git a/flaml/nlp/huggingface/training_args.py b/flaml/nlp/huggingface/training_args.pyindex 3f5efe61a..e65b3bb0a 100644--- a/flaml/nlp/huggingface/training_args.py+++ b/flaml/nlp/huggingface/training_args.py@@ -17,13 +17,21 @@ class TrainingArgumentsForAuto(TrainingArguments):"""FLAML custom TrainingArguments.Args:+ task (str): the task name for NLP tasks, e.g., seq-classification, token-classificationoutput_dir (str): data root directory for outputing the log, etc.model_path (str, optional, defaults to "facebook/muppet-roberta-base"): A string,the path of the language model file, either a path from huggingfacemodel card huggingface.co/models, or a local path for the model.fp16 (bool, optional, defaults to "False"): A bool, whether to use FP16.max_seq_length (int, optional, defaults to 128): An integer, the max length of the sequence.+ pad_to_max_length (bool, optional, defaults to "False"):+ whether to pad all samples to model maximum sentence length.+ If False, will pad the samples dynamically when batching to the maximum length in the batch.ckpt_per_epoch (int, optional, defaults to 1): An integer, the number of checkpoints per epoch.+ per_device_eval_batch_size (int, optional, defaults to 1): An integer, the per gpu evaluation batch size.+ label_list (List[str], optional, defaults to None): A list of string, the string list of the label names.+ When the task is sequence labeling/token classification, need to set the label_list (e.g., B-PER, I-PER, B-LOC)+ to obtain the correct evaluation metric. See the example in test/nlp/test_autohf_tokenclassification.py."""task: str = field(default="seq-classification")@@ -37,21 +45,15 @@ class TrainingArgumentsForAuto(TrainingArguments):},)- tokenizer_model_path: str = field(- default=None,- metadata={"help": "tokenizer model path for HPO"},- )-fp16: bool = field(default=True, metadata={"help": "whether to use the FP16 mode"})max_seq_length: int = field(default=128, metadata={"help": "max seq length"})pad_to_max_length: bool = field(- default=True,+ default=False,metadata={"help": "Whether to pad all samples to model maximum sentence length. "- "If False, will pad the samples dynamically when batching to the maximum length in the batch. More "- "efficient on GPU but very bad for TPU."+ "If False, will pad the samples dynamically when batching to the maximum length in the batch. "},)@@ -62,21 +64,8 @@ class TrainingArgumentsForAuto(TrainingArguments):metadata={"help": "per gpu evaluation batch size"},)- report_to: Optional[List[str]] = field(- default=None,- metadata={- "help": "The list of integrations to report the results and logs to."- },- )-- do_train: bool = field(default=False, metadata={"help": "Whether to run training."})- do_eval: bool = field(- default=False, metadata={"help": "Whether to run eval on the dev set."}- )-- metric_for_best_model: Optional[str] = field(- default="loss",- metadata={"help": "The metric to use to compare two different models."},+ label_list: Optional[List[str]] = field(+ default=None, metadata={"help": "The string list of the label names. "})@staticmethoddiff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.pyindex ab26eb803..cd2e7a409 100644--- a/flaml/nlp/utils.py+++ b/flaml/nlp/utils.py@@ -83,7 +83,9 @@ def tokenize_and_align_labels():tokenized_inputs = tokenizer([list(examples[X_sent_key])],- padding="max_length",+ padding="max_length"+ if hf_args.pad_to_max_length+ else False, # to be consistent with https://github.com/huggingface/transformers/blob/main/examples/pytorch/token-classification/run_ner.py#L394truncation=True,max_length=hf_args.max_seq_length,# We use this argument because the texts in our dataset are lists of words (with a label for each word).@@ -113,11 +115,11 @@ def tokenize_and_align_labels(# else:# label_ids.append(b_to_i_label[label_to_id[label[word_idx]]])previous_word_idx = word_idx- tokenized_inputs["label"] = label_ids+ tokenized_inputs["labels"] = label_idstmp_column_names = sorted(tokenized_inputs.keys())tokenized_input_and_labels = [tokenized_inputs[x] for x in tmp_column_names]for key_idx, each_key in enumerate(tmp_column_names):- if each_key != "label":+ if each_key != "labels":tokenized_input_and_labels[key_idx] = tokenized_input_and_labels[key_idx][0]if return_column_name:return tokenized_input_and_labels, tmp_column_names@@ -151,7 +153,7 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):axis=1,result_type="expand",)- label_idx = tokenized_column_names.index("label")+ label_idx = tokenized_column_names.index("labels")other_indices = sorted(set(range(len(tokenized_column_names))).difference({label_idx}))diff --git a/test/automl/test_classification.py b/test/automl/test_classification.pyindex 6ca9db785..cb09b13ff 100644--- a/test/automl/test_classification.py+++ b/test/automl/test_classification.py@@ -347,3 +347,4 @@ class TestClassification(unittest.TestCase):if __name__ == "__main__":unittest.main()+ test = TestClassification()diff --git a/test/nlp/test_autohf_tokenclassification.py b/test/nlp/test_autohf_tokenclassification.pyindex 9fb3a663a..7636193ce 100644--- a/test/nlp/test_autohf_tokenclassification.py+++ b/test/nlp/test_autohf_tokenclassification.py@@ -13,7 +13,18 @@ def test_tokenclassification():automl_settings = get_automl_settings()automl_settings["task"] = "token-classification"- automl_settings["metric"] = "seqeval"+ automl_settings["metric"] = "seqeval:overall_f1" # evaluating based on the overall_f1 of seqeval+ automl_settings["fit_kwargs_by_estimator"]["transformer"]["label_list"] = [+ "O",+ "B-PER",+ "I-PER",+ "B-ORG",+ "I-ORG",+ "B-LOC",+ "I-LOC",+ "B-MISC",+ "I-MISC",+ ]try:automl.fit(