A programming framework for agentic AI
fix ner bug; refactor post processing of TransformersEstimator prediction (#615)
Xueqing Liu6108493e0b77
diff --git a/flaml/automl.py b/flaml/automl.pyindex 37650d359..3b065504a 100644--- a/flaml/automl.py+++ b/flaml/automl.py@@ -1108,10 +1108,21 @@ class AutoML(BaseEstimator):from .data import DataTransformerself._transformer = DataTransformer()+self._X_train_all, self._y_train_all = self._transformer.fit_transform(X, y, self._state.task)self._label_transformer = self._transformer.label_transformer+ if hasattr(self._label_transformer, "label_list"):+ self._state.fit_kwargs.update(+ {"label_list": self._label_transformer.label_list}+ )+ elif self._state.task == TOKENCLASSIFICATION:+ if "label_list" not in self._state.fit_kwargs:+ for each_fit_kwargs in self._state.fit_kwargs_by_estimator.values():+ assert (+ "label_list" in each_fit_kwargs+ ), "For the token-classification task, you must either (1) pass token labels; or (2) pass id labels and the label list. Please refer to the documentation for more details: https://microsoft.github.io/FLAML/docs/Examples/AutoML-NLP#a-simple-token-classification-example"self._sample_weight_full = self._state.fit_kwargs.get("sample_weight"diff --git a/flaml/data.py b/flaml/data.pyindex 379147ac0..28960a0e2 100644--- a/flaml/data.py+++ b/flaml/data.py@@ -371,14 +371,20 @@ class DataTransformer:)self._drop = dropif (- (task in CLASSIFICATION or not pd.api.types.is_numeric_dtype(y))+ task in CLASSIFICATION+ or not pd.api.types.is_numeric_dtype(y)and task not in NLG_TASKS- and task != TOKENCLASSIFICATION):- from sklearn.preprocessing import LabelEncoder+ if task != TOKENCLASSIFICATION:+ from sklearn.preprocessing import LabelEncoder- self.label_transformer = LabelEncoder()+ self.label_transformer = LabelEncoder()+ else:+ from .nlp.utils import LabelEncoderforTokenClassification++ self.label_transformer = LabelEncoderforTokenClassification()y = self.label_transformer.fit_transform(y)+else:self.label_transformer = Noneself._task = taskdiff --git a/flaml/ml.py b/flaml/ml.pyindex 75d67f4be..49c0b0bf0 100644--- a/flaml/ml.py+++ b/flaml/ml.py@@ -134,15 +134,22 @@ def get_estimator_class(task, estimator_name):def metric_loss_score(metric_name,- y_predict,- y_true,+ y_processed_predict,+ y_processed_true,labels=None,sample_weight=None,groups=None,):+ # y_processed_predict and y_processed_true are processed id labels if the original were the token labels+if is_in_sklearn_metric_name_set(metric_name):return sklearn_metric_loss_score(- metric_name, y_predict, y_true, labels, sample_weight, groups+ metric_name,+ y_processed_predict,+ y_processed_true,+ labels,+ sample_weight,+ groups,)else:"""@@ -151,14 +158,22 @@ def metric_loss_score(if metric_name == "spearmanr":from scipy.stats import spearmanr- y_true = y_true.to_list() if type(y_true) == pd.Series else list(y_true)- score = spearmanr(list(y_predict), y_true)[0]+ y_true = (+ y_processed_true.to_list()+ if isinstance(y_processed_true, pd.Series)+ else list(y_processed_true)+ )+ score = spearmanr(list(y_processed_predict), y_true)[0]metric_mode = "max"elif metric_name == "pearsonr":from scipy.stats import pearsonr- y_true = y_true.to_list() if type(y_true) == pd.Series else list(y_true)- score = pearsonr(list(y_predict), y_true)[0]+ y_true = (+ y_processed_true.to_list()+ if type(y_processed_true) == pd.Series+ else list(y_processed_true)+ )+ score = pearsonr(list(y_processed_predict), y_true)[0]metric_mode = "max"else:try:@@ -171,43 +186,31 @@ def metric_loss_score(metric_mode = huggingface_metric_to_mode[datasets_metric_name]if "rouge" in metric_name:- score = metric.compute(predictions=y_predict, references=y_true)[- metric_name- ].mid.fmeasure+ score = metric.compute(+ predictions=y_processed_predict, references=y_processed_true+ )[metric_name].mid.fmeasureelif metric_name.startswith("seqeval"):- label_len = len(labels)- zip_pred_true = [- [(p, lb) for (p, lb) in zip(prediction, label) if lb != -100]- for (prediction, label) in zip(y_predict, y_true)+ y_processed_true = [+ [labels[tr] for tr in each_list]+ for each_list in y_processed_true]- y_pred = [- [- labels[p] if 0 <= p < label_len else -1- for (p, l) in each_list- ]- for each_list in zip_pred_true- ] # To compute precision and recall, y_pred and y_true must be converted to string labels- # (B-PER, I-PER, etc.), so that the category-based precision/recall (i.e., PER, LOC, etc.) scores can be computed- y_true = [- [labels[l] for (p, l) in each_list]- for each_list in zip_pred_true- ]-metric_submetric_names = metric_name.split(":")- score = metric.compute(predictions=y_pred, references=y_true)[+ score = metric.compute(+ predictions=y_processed_predict, references=y_processed_true+ )[metric_submetric_names[1]if len(metric_submetric_names) > 1else "overall_accuracy"]else:- score = metric.compute(predictions=y_predict, references=y_true)[- metric_name- ]+ score = metric.compute(+ predictions=y_processed_predict, references=y_processed_true+ )[metric_name]except ImportError:- raise Exception(+ raise ValueError(metric_name+ " is not an built-in sklearn metric and nlp is not installed. ""Currently built-in sklearn metrics are: "@@ -219,7 +222,7 @@ def metric_loss_score(# If the metric is not found from huggingface dataset metric list (i.e., FileNotFoundError)# ask the user to provide a custom metricexcept FileNotFoundError:- raise Exception(+ raise ValueError(metric_name+ " is neither an sklearn metric nor a huggingface metric. ""Currently built-in sklearn metrics are: "@@ -265,14 +268,13 @@ def sklearn_metric_loss_score(used to calculate the metric. E.g., 2d for log_loss and 1dfor others.y_true: A 1d numpy array of the true labels.- labels: A 1d numpy array of the unique labels.+ labels: A list or an array of the unique labels.sample_weight: A 1d numpy array of the sample weight.groups: A 1d numpy array of the group labels.Returns:score: A float number of the loss, the lower the better."""-metric_name = metric_name.lower()if "r2" == metric_name:@@ -368,8 +370,14 @@ def _eval_estimator(pred_start = time.time()val_pred_y = get_y_pred(estimator, X_val, eval_metric, obj)pred_time = (time.time() - pred_start) / X_val.shape[0]+val_loss = metric_loss_score(- eval_metric, val_pred_y, y_val, labels, weight_val, groups_val+ eval_metric,+ y_processed_predict=val_pred_y,+ y_processed_true=y_val,+ labels=labels,+ sample_weight=weight_val,+ groups=groups_val,)metric_for_logging = {"pred_time": pred_time}if log_training_metric:@@ -588,6 +596,7 @@ def compute_estimator()if isinstance(estimator, TransformersEstimator):+ # TODO: move the partial function to nlpfit_kwargs["metric"] = eval_metricfit_kwargs["X_val"] = X_valfit_kwargs["y_val"] = y_valdiff --git a/flaml/model.py b/flaml/model.pyindex f73187898..f6318306c 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -522,14 +522,12 @@ class TransformersEstimator(BaseEstimator):@propertydef num_labels(self):- from .data import SEQCLASSIFICATION, SEQREGRESSION, TOKENCLASSIFICATION-if self._task == SEQREGRESSION:return 1elif self._task == SEQCLASSIFICATION:return len(set(self._y_train))elif self._task == TOKENCLASSIFICATION:- return len(set([a for b in self._y_train.tolist() for a in b]))+ return len(self._training_args.label_list)else:return None@@ -549,9 +547,9 @@ class TransformersEstimator(BaseEstimator):return AutoTokenizer.from_pretrained(self._training_args.model_path,use_fast=True,- add_prefix_space=True- if "roberta" in self._training_args.model_path- else False, # If roberta model, must set add_prefix_space to True to avoid the assertion error at+ add_prefix_space="roberta" in self._training_args.model_path+ and not getattr(self, "_pred_flag", False),+ # If roberta model and the call is from .fit instead of .predict (when the model_path is updated to the checkpoint name instead), must set add_prefix_space to True to avoid the assertion error at# https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249)@@ -734,39 +732,30 @@ class TransformersEstimator(BaseEstimator):return best_ckptdef _compute_metrics_by_dataset_name(self, eval_pred):+ # TODO: call self._metric(eval_pred, self)if isinstance(self._metric, str):from .ml import metric_loss_score- from .nlp.utils import postprocess_text-- predictions, labels = eval_pred- if self._task in NLG_TASKS:- if isinstance(predictions, tuple):- predictions = np.argmax(predictions[0], axis=2)- decoded_preds = self.tokenizer.batch_decode(- predictions, skip_special_tokens=True- )- labels = np.where(labels != -100, labels, self.tokenizer.pad_token_id)- decoded_labels = self.tokenizer.batch_decode(- labels, skip_special_tokens=True- )- predictions, labels = postprocess_text(decoded_preds, decoded_labels)- else:- predictions = (- np.squeeze(predictions)- if self._task == SEQREGRESSION- else np.argmax(predictions, axis=2)- if self._task == TOKENCLASSIFICATION- else np.argmax(predictions, axis=1)- )+ from .nlp.utils import postprocess_prediction_and_true++ predictions, y_true = eval_pred+ # postprocess the matrix prediction and ground truth into user readable format, e.g., for summarization, decode into text+ processed_predictions, processed_y_true = postprocess_prediction_and_true(+ task=self._task,+ y_pred=predictions,+ tokenizer=self.tokenizer,+ hf_args=self._training_args,+ y_true=y_true,+ )metric_dict = {"automl_metric": metric_loss_score(metric_name=self._metric,- y_predict=predictions,- y_true=labels,+ y_processed_predict=processed_predictions,+ y_processed_true=processed_y_true,labels=self._training_args.label_list,)}else:+ # TODO: debug to see how custom metric can take both tokenized (here) and untokenized input (ml.py)loss, metric_dict = self._metric(X_test=self._X_val,y_test=self._y_val,@@ -786,6 +775,7 @@ class TransformersEstimator(BaseEstimator):Need to reinit training_args because of a bug in deepspeed: if not reinit, the deepspeed config will be inconsistentwith HF config https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py#L947"""+ self._pred_flag = Truetraining_args = self._TrainingArguments(local_rank=-1, model_path=self._checkpoint_path, fp16=self.fp16)@@ -837,6 +827,7 @@ class TransformersEstimator(BaseEstimator):def predict(self, X, **pred_kwargs):import transformersfrom datasets import Dataset+ from .nlp.utils import postprocess_prediction_and_truetransformers.logging.set_verbosity_error()@@ -856,20 +847,14 @@ class TransformersEstimator(BaseEstimator):test_dataset,metric_key_prefix="predict",)-- if self._task == SEQCLASSIFICATION:- return np.argmax(predictions.predictions, axis=1)- elif self._task == SEQREGRESSION:- return predictions.predictions.reshape((len(predictions.predictions),))- elif self._task == TOKENCLASSIFICATION:- return np.argmax(predictions.predictions, axis=2)- elif self._task == SUMMARIZATION:- decoded_preds = self.tokenizer.batch_decode(- predictions.predictions, skip_special_tokens=True- )- return decoded_preds- elif self._task == MULTICHOICECLASSIFICATION:- return np.argmax(predictions.predictions, axis=1)+ post_y_pred, _ = postprocess_prediction_and_true(+ task=self._task,+ y_pred=predictions.predictions,+ tokenizer=self.tokenizer,+ hf_args=self._training_args,+ X=X,+ )+ return post_y_preddef config2params(self, config: dict) -> dict:params = super().config2params(config)diff --git a/flaml/nlp/huggingface/training_args.py b/flaml/nlp/huggingface/training_args.pyindex e65b3bb0a..8790d2a5f 100644--- a/flaml/nlp/huggingface/training_args.py+++ b/flaml/nlp/huggingface/training_args.py@@ -30,8 +30,10 @@ class TrainingArgumentsForAuto(TrainingArguments):ckpt_per_epoch (int, optional, defaults to 1): An integer, the number of checkpoints per epoch.per_device_eval_batch_size (int, optional, defaults to 1): An integer, the per gpu evaluation batch size.label_list (List[str], optional, defaults to None): A list of string, the string list of the label names.- When the task is sequence labeling/token classification, need to set the label_list (e.g., B-PER, I-PER, B-LOC)- to obtain the correct evaluation metric. See the example in test/nlp/test_autohf_tokenclassification.py.+ When the task is sequence labeling/token classification, there are two formats of the labels:+ (1) The token labels, i.e., [B-PER, I-PER, B-LOC]; (2) Id labels. For (2), need to pass the label_list (e.g., [B-PER, I-PER, B-LOC])+ to convert the Id to token labels when computing the metric with metric_loss_score.+ See the example in [a simple token classification example](../../../Examples/AutoML-NLP#a-simple-token-classification-example)."""task: str = field(default="seq-classification")@@ -49,6 +51,13 @@ class TrainingArgumentsForAuto(TrainingArguments):max_seq_length: int = field(default=128, metadata={"help": "max seq length"})+ label_all_tokens: bool = field(+ default=False,+ metadata={+ "help": "For NER task, whether to set the extra tokenized labels to the same label (instead of -100)"+ },+ )+pad_to_max_length: bool = field(default=False,metadata={diff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.pyindex 390b05c71..5f0d1464e 100644--- a/flaml/nlp/utils.py+++ b/flaml/nlp/utils.py@@ -11,6 +11,8 @@ from ..data import (NLG_TASKS,)+import pandas as pd+def load_default_huggingface_metric_for_task(task):@@ -54,28 +56,30 @@ def tokenize_seq2seq(X, Y, tokenizer, task=None, hf_args=None):hf_args=hf_args,prefix_str="summarize: ",)- labels = None+ model_outputs = Noneif Y is not None:- labels = tokenize_onedataframe(+ model_outputs = tokenize_onedataframe(Y.to_frame(),tokenizer=tokenizer,task=task,hf_args=hf_args,prefix_str="",)- labels["label"] = [+ model_outputs["label"] = [[(each_l if each_l != tokenizer.pad_token_id else -100) for each_l in label]- for label in labels["input_ids"]+ for label in model_outputs["input_ids"]]- labels = labels.drop(+ model_outputs = model_outputs.drop(columns=["attention_mask", "input_ids", "decoder_input_ids"])- return model_inputs, labels+ return model_inputs, model_outputsdef tokenize_and_align_labels(examples,tokenizer,+ label_to_id,+ b_to_i_label,hf_args=None,X_sent_key=None,Y_sent_key=None,@@ -94,26 +98,22 @@ def tokenize_and_align_labels(if Y_sent_key is not None:previous_word_idx = Nonelabel_ids = []- import numbers-for word_idx in tokenized_inputs.word_ids(batch_index=0):- # Special tokens have a word id that is None. We set the label to -100 so they are automatically- # ignored in the loss function.if word_idx is None:label_ids.append(-100)- # We set the label for the first token of each word.elif word_idx != previous_word_idx:- if isinstance(examples[Y_sent_key][word_idx], numbers.Number):- label_ids.append(examples[Y_sent_key][word_idx])- # else:- # label_ids.append(label_to_id[label[word_idx]])+ label_ids.append(label_to_id[examples[Y_sent_key][word_idx]])# For the other tokens in a word, we set the label to either the current label or -100, depending on# the label_all_tokens flag.else:- if isinstance(examples[Y_sent_key][word_idx], numbers.Number):- label_ids.append(examples[Y_sent_key][word_idx])- # else:- # label_ids.append(b_to_i_label[label_to_id[label[word_idx]]])+ # Use the label_all_tokens to control whether to copy the label to all subtokens or to pad the additional tokens as -100+ if hf_args.label_all_tokens:+ # If the B- word is converted into multiple subtokens, map the additional subtokens to I-+ label_ids.append(+ b_to_i_label[label_to_id[examples[Y_sent_key][word_idx]]]+ )+ else:+ label_ids.append(-100)previous_word_idx = word_idxtokenized_inputs["labels"] = label_idstmp_column_names = sorted(tokenized_inputs.keys())@@ -128,7 +128,15 @@ def tokenize_and_align_labels(def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):- import pandas as pd++ # If the label_all_tokens flag is True, prepare two dicts label_to_id and b_to_i_label to convert the B- labels to I- labels+ label_to_id = {i: i for i in range(len(hf_args.label_list))}+ b_to_i_label = []+ for idx, label in enumerate(hf_args.label_list):+ if label.startswith("B-") and label.replace("B-", "I-") in hf_args.label_list:+ b_to_i_label.append(hf_args.label_list.index(label.replace("B-", "I-")))+ else:+ b_to_i_label.append(idx)if Y is not None:X_and_Y = pd.concat([X, Y.to_frame()], axis=1)@@ -141,6 +149,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):X_sent_key=X_key,Y_sent_key=Y_key,return_column_name=True,+ label_to_id=label_to_id,+ b_to_i_label=b_to_i_label,)X_and_Y_tokenized = X_and_Y.apply(lambda x: tokenize_and_align_labels(@@ -149,6 +159,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):hf_args=hf_args,X_sent_key=X_key,Y_sent_key=Y_key,+ label_to_id=label_to_id,+ b_to_i_label=b_to_i_label,),axis=1,result_type="expand",@@ -170,6 +182,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):X_sent_key=X_key,Y_sent_key=None,return_column_name=True,+ label_to_id=label_to_id,+ b_to_i_label=b_to_i_label,)d = X.apply(@@ -179,6 +193,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):hf_args=hf_args,X_sent_key=X_key,Y_sent_key=None,+ label_to_id=label_to_id,+ b_to_i_label=b_to_i_label,),axis=1,result_type="expand",@@ -197,7 +213,6 @@ def tokenize_onedataframe(hf_args=None,prefix_str=None,):- import pandaswith tokenizer.as_target_tokenizer():_, tokenized_column_names = tokenize_row(@@ -219,25 +234,11 @@ def tokenize_onedataframe(axis=1,result_type="expand",)- X_tokenized = pandas.DataFrame(columns=tokenized_column_names)+ X_tokenized = pd.DataFrame(columns=tokenized_column_names)X_tokenized[tokenized_column_names] = dreturn X_tokenized-def postprocess_text(preds, labels):- import nltk-- nltk.download("punkt")- preds = [pred.strip() for pred in preds]- labels = [label.strip() for label in labels]-- # rougeLSum expects newline after each sentence- preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds]- labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels]-- return preds, labels--def tokenize_row(this_row,tokenizer,@@ -267,7 +268,6 @@ def tokenize_row(def tokenize_text_multiplechoice(X, tokenizer, hf_args=None):- import pandast = X[["sent1", "sent2", "ending0", "ending1", "ending2", "ending3"]]_, tokenized_column_names = tokenize_swag(@@ -282,7 +282,7 @@ def tokenize_text_multiplechoice(X, tokenizer, hf_args=None):result_type="expand",)- X_tokenized = pandas.DataFrame(columns=tokenized_column_names)+ X_tokenized = pd.DataFrame(columns=tokenized_column_names)X_tokenized[tokenized_column_names] = doutput = X_tokenized.join(X)return output, None@@ -475,3 +475,98 @@ def load_model(checkpoint_path, task, num_labels=None):model_config = _set_model_config(checkpoint_path)this_model = get_this_model(checkpoint_path, task, model_config)return this_model+++def postprocess_prediction_and_true(+ task, y_pred, tokenizer, hf_args, y_true=None, X=None+):+ # postprocess the matrix prediction y_pred and ground truth y_true into user readable format, e.g., for summarization, decode into text+ if task == SEQCLASSIFICATION:+ return np.argmax(y_pred, axis=1), y_true+ elif task == SEQREGRESSION:+ return np.squeeze(y_pred), y_true # predictions.reshape((len(predictions),))+ elif task == TOKENCLASSIFICATION:+ assert (y_true is not None) or (+ X is not None+ ), "One of y_true and X must not be None"+ ## If y_true is not None, we use y_true to remove the -100 in the prediction (postprocessing), and return the postprocessed y_true and prediction+ # If y_true is None, we use X to compute y_is_pad (i.e., whether y_true is -100 in that position), and use y_is_pad to remove the -100 in the prediction, and return the postprocessed prediction (not the y_true)+ y_predict = pd.Series(np.argmax(y_pred, axis=2).tolist())+ if y_true is None:+ _, y_is_pad = tokenize_text(+ X,+ y_predict,+ task=task,+ hf_args=hf_args,+ tokenizer=tokenizer,+ )+ else:+ y_is_pad = y_true+ label_len = len(hf_args.label_list)+ zip_pred_ispad = [+ [(p, ispd) for (p, ispd) in zip(each_pred, each_is_pad) if ispd != -100]+ for (each_pred, each_is_pad) in zip(y_predict, y_is_pad)+ ]+ y_pred_label = [+ [+ hf_args.label_list[p] if 0 <= p < label_len else -1+ for (p, ispd) in each_list+ ]+ for each_list in zip_pred_ispad+ ] # To compute precision and recall, y_pred and y_true must be converted to string labels+ # (B-PER, I-PER, etc.), so that the category-based precision/recall (i.e., PER, LOC, etc.) scores can be computed+ if y_true is not None:+ y_true_label = [+ [tr for (p, tr) in each_list] for each_list in zip_pred_ispad+ ]+ else:+ y_true_label = None+ return y_pred_label, y_true_label+ elif task == SUMMARIZATION:+ if isinstance(y_pred, tuple):+ y_pred = np.argmax(y_pred[0], axis=2)+ decoded_preds = tokenizer.batch_decode(y_pred, skip_special_tokens=True)++ import nltk++ nltk.download("punkt")+ decoded_preds = [pred.strip() for pred in decoded_preds]+ decoded_preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in decoded_preds]++ if y_true is not None:+ y_true_labels = np.where(y_true != -100, y_true, tokenizer.pad_token_id)+ decoded_y_true_labels = tokenizer.batch_decode(+ y_true_labels, skip_special_tokens=True+ )+ decoded_y_true_labels = [label.strip() for label in decoded_y_true_labels]+ decoded_y_true_labels = [+ "\n".join(nltk.sent_tokenize(label)) for label in decoded_y_true_labels+ ]+ else:+ decoded_y_true_labels = None++ return decoded_preds, decoded_y_true_labels+ elif task == MULTICHOICECLASSIFICATION:+ return np.argmax(y_pred, axis=1), y_true+++class LabelEncoderforTokenClassification:+ def fit_transform(self, y):+ # if the labels are tokens, convert them to ids+ if any(isinstance(id, str) for id in y[0]):+ self.label_list = sorted(list(set().union(*y)))+ self._tokenlabel_to_id = {+ self.label_list[id]: id for id in range(len(self.label_list))+ }+ y = y.apply(lambda sent: [self._tokenlabel_to_id[token] for token in sent])+ # if the labels are not tokens, they must be ids+ else:+ assert all(+ isinstance(id, int) for id in y[0]+ ), "The labels must either be tokens or ids"+ return y++ def transform(self, y):+ if hasattr(self, "_tokenlabel_to_id"):+ y = y.apply(lambda sent: [self._tokenlabel_to_id[token] for token in sent])+ return ydiff --git a/test/nlp/test_autohf_custom_metric.py b/test/nlp/test_autohf_custom_metric.pyindex 3b8e0a360..ee5317d15 100644--- a/test/nlp/test_autohf_custom_metric.py+++ b/test/nlp/test_autohf_custom_metric.py@@ -25,10 +25,10 @@ def custom_metric(else:trainer = estimator._trainerif y_test is not None:- X_test, _ = estimator._preprocess(X_test)+ X_test = estimator._preprocess(X_test)eval_dataset = Dataset.from_pandas(TransformersEstimator._join(X_test, y_test))else:- X_test, _ = estimator._preprocess(X_test)+ X_test = estimator._preprocess(X_test)eval_dataset = Dataset.from_pandas(X_test)estimator_metric_backup = estimator._metricdiff --git a/test/nlp/test_autohf_summarization.py b/test/nlp/test_autohf_summarization.pyindex ac05a7f2f..e52a8754b 100644--- a/test/nlp/test_autohf_summarization.py+++ b/test/nlp/test_autohf_summarization.py@@ -6,6 +6,7 @@ from utils import get_toy_data_summarization, get_automl_settings@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os")def test_summarization():+ # TODO: manual test for how effective postprocess_seq2seq_prediction_label isfrom flaml import AutoMLX_train, y_train, X_val, y_val, X_test = get_toy_data_summarization()diff --git a/test/nlp/test_autohf_tokenclassification.py b/test/nlp/test_autohf_tokenclassification.pyindex 2a60617e9..cb1c40e9d 100644--- a/test/nlp/test_autohf_tokenclassification.py+++ b/test/nlp/test_autohf_tokenclassification.py@@ -1,17 +1,21 @@import sysimport pytestimport requests-from utils import get_toy_data_tokenclassification, get_automl_settings+from utils import (+ get_toy_data_tokenclassification_idlabel,+ get_toy_data_tokenclassification_tokenlabel,+ get_automl_settings,+)@pytest.mark.skipif(sys.platform == "darwin" or sys.version < "3.7",reason="do not run on mac os or py<3.7",)-def test_tokenclassification():+def test_tokenclassification_idlabel():from flaml import AutoML- X_train, y_train, X_val, y_val = get_toy_data_tokenclassification()+ X_train, y_train, X_val, y_val = get_toy_data_tokenclassification_idlabel()automl = AutoML()automl_settings = get_automl_settings()@@ -42,6 +46,66 @@ def test_tokenclassification():except requests.exceptions.HTTPError:return+ # perf test+ import json++ with open("seqclass.log", "r") as fin:+ for line in fin:+ each_log = json.loads(line.strip("\n"))+ if "validation_loss" in each_log:+ val_loss = each_log["validation_loss"]+ min_inter_result = min(+ each_dict.get("eval_automl_metric", sys.maxsize)+ for each_dict in each_log["logged_metric"]["intermediate_results"]+ )++ if min_inter_result != sys.maxsize:+ assert val_loss == min_inter_result+++ sys.platform == "darwin" or sys.version < "3.7",+ reason="do not run on mac os or py<3.7",+)+def test_tokenclassification_tokenlabel():+ from flaml import AutoML++ X_train, y_train, X_val, y_val = get_toy_data_tokenclassification_tokenlabel()+ automl = AutoML()++ automl_settings = get_automl_settings()+ automl_settings["task"] = "token-classification"+ automl_settings[+ "metric"+ ] = "seqeval:overall_f1" # evaluating based on the overall_f1 of seqeval++ try:+ automl.fit(+ X_train=X_train,+ y_train=y_train,+ X_val=X_val,+ y_val=y_val,+ **automl_settings+ )+ except requests.exceptions.HTTPError:+ return++ # perf test+ import json++ with open("seqclass.log", "r") as fin:+ for line in fin:+ each_log = json.loads(line.strip("\n"))+ if "validation_loss" in each_log:+ val_loss = each_log["validation_loss"]+ min_inter_result = min(+ each_dict.get("eval_automl_metric", sys.maxsize)+ for each_dict in each_log["logged_metric"]["intermediate_results"]+ )++ if min_inter_result != sys.maxsize:+ assert val_loss == min_inter_result+if __name__ == "__main__":- test_tokenclassification()+ test_tokenclassification_idlabel()diff --git a/test/nlp/utils.py b/test/nlp/utils.pyindex 8d07bada1..af512c90e 100644--- a/test/nlp/utils.py+++ b/test/nlp/utils.py@@ -406,7 +406,8 @@ def get_toy_data_summarization():return X_train, y_train, X_val, y_val, X_test-def get_toy_data_tokenclassification():+def get_toy_data_tokenclassification_idlabel():+ # test token classification when the labels are idstrain_data = {"chunk_tags": [[11, 21, 11, 12, 21, 22, 11, 12, 0],@@ -1116,6 +1117,391 @@ def get_toy_data_tokenclassification():return X_train, y_train, X_val, y_val+def get_toy_data_tokenclassification_tokenlabel():+ # test token classification when the labels are tokens+ train_data = {+ "id": ["0", "1", "2", "3"],+ "ner_tags": [+ ["B-ORG", "O", "B-MISC", "O", "O", "O", "B-MISC", "O", "O"],+ ["B-PER", "I-PER"],+ ["B-LOC", "O"],+ [+ "O",+ "B-ORG",+ "I-ORG",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "B-MISC",+ "O",+ "O",+ "O",+ "O",+ "O",+ "B-MISC",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ ],+ ],+ "tokens": [+ [+ "EU",+ "rejects",+ "German",+ "call",+ "to",+ "boycott",+ "British",+ "lamb",+ ".",+ ],+ ["Peter", "Blackburn"],+ ["BRUSSELS", "1996-08-22"],+ [+ "The",+ "European",+ "Commission",+ "said",+ "on",+ "Thursday",+ "it",+ "disagreed",+ "with",+ "German",+ "advice",+ "to",+ "consumers",+ "to",+ "shun",+ "British",+ "lamb",+ "until",+ "scientists",+ "determine",+ "whether",+ "mad",+ "cow",+ "disease",+ "can",+ "be",+ "transmitted",+ "to",+ "sheep",+ ".",+ ],+ ],+ }++ dev_data = {+ "id": ["4", "5", "6", "7"],+ "ner_tags": [+ [+ "B-LOC",+ "O",+ "O",+ "O",+ "O",+ "B-ORG",+ "I-ORG",+ "O",+ "O",+ "O",+ "B-PER",+ "I-PER",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "B-LOC",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ ],+ [+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "B-ORG",+ "O",+ "O",+ "O",+ "B-PER",+ "I-PER",+ "I-PER",+ "I-PER",+ "O",+ "O",+ "O",+ "O",+ "O",+ ],+ [+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "B-ORG",+ "I-ORG",+ "O",+ ],+ [+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "B-ORG",+ "O",+ "O",+ "B-PER",+ "I-PER",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ "O",+ ],+ ],+ "tokens": [+ [+ "Germany",+ "'s",+ "representative",+ "to",+ "the",+ "European",+ "Union",+ "'s",+ "veterinary",+ "committee",+ "Werner",+ "Zwingmann",+ "said",+ "on",+ "Wednesday",+ "consumers",+ "should",+ "buy",+ "sheepmeat",+ "from",+ "countries",+ "other",+ "than",+ "Britain",+ "until",+ "the",+ "scientific",+ "advice",+ "was",+ "clearer",+ ".",+ ],+ [+ '"',+ "We",+ "do",+ "n't",+ "support",+ "any",+ "such",+ "recommendation",+ "because",+ "we",+ "do",+ "n't",+ "see",+ "any",+ "grounds",+ "for",+ "it",+ ",",+ '"',+ "the",+ "Commission",+ "'s",+ "chief",+ "spokesman",+ "Nikolaus",+ "van",+ "der",+ "Pas",+ "told",+ "a",+ "news",+ "briefing",+ ".",+ ],+ [+ "He",+ "said",+ "further",+ "scientific",+ "study",+ "was",+ "required",+ "and",+ "if",+ "it",+ "was",+ "found",+ "that",+ "action",+ "was",+ "needed",+ "it",+ "should",+ "be",+ "taken",+ "by",+ "the",+ "European",+ "Union",+ ".",+ ],+ [+ "He",+ "said",+ "a",+ "proposal",+ "last",+ "month",+ "by",+ "EU",+ "Farm",+ "Commissioner",+ "Franz",+ "Fischler",+ "to",+ "ban",+ "sheep",+ "brains",+ ",",+ "spleens",+ "and",+ "spinal",+ "cords",+ "from",+ "the",+ "human",+ "and",+ "animal",+ "food",+ "chains",+ "was",+ "a",+ "highly",+ "specific",+ "and",+ "precautionary",+ "move",+ "to",+ "protect",+ "human",+ "health",+ ".",+ ],+ ],+ }+ train_dataset = pd.DataFrame(train_data)+ dev_dataset = pd.DataFrame(dev_data)++ custom_sent_keys = ["tokens"]+ label_key = "ner_tags"++ X_train = train_dataset[custom_sent_keys]+ y_train = train_dataset[label_key]++ X_val = dev_dataset[custom_sent_keys]+ y_val = dev_dataset[label_key]+ return X_train, y_train, X_val, y_val++def get_automl_settings(estimator_name="transformer"):automl_settings = {diff --git a/website/docs/Examples/AutoML-NLP.md b/website/docs/Examples/AutoML-NLP.mdindex 8976345cd..df3b971aa 100644--- a/website/docs/Examples/AutoML-NLP.md+++ b/website/docs/Examples/AutoML-NLP.md@@ -26,7 +26,7 @@ automl = AutoML()automl_settings = {"time_budget": 100,"task": "seq-classification",- "fit_kwargs_by_estimator": {+ "fit_kwargs_by_estimator": {"transformer":{"output_dir": "data/output/" # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base@@ -87,7 +87,7 @@ automl_settings["fit_kwargs_by_estimator"] = { # setting the huggingface argume"model_path": "google/electra-small-discriminator", # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base"output_dir": "data/output/", # setting the output directory"ckpt_per_epoch": 5, # setting the number of checkpoints per epoch- "fp16": False,+ "fp16": False,} # setting whether to use FP16}automl.fit(@@ -139,7 +139,7 @@ automl_settings["fit_kwargs_by_estimator"] = { # setting the huggingface ar"model_path": "t5-small", # if model_path is not set, the default model is t5-small: https://huggingface.co/t5-small"output_dir": "data/output/", # setting the output directory"ckpt_per_epoch": 5, # setting the number of checkpoints per epoch- "fp16": False,+ "fp16": False,} # setting whether to use FP16}automl.fit(@@ -214,6 +214,154 @@ Model config T5Config {}```+### A simple token classification example++There are two ways to define the label for a token classification task. The first is to define the token labels:++```python+from flaml import AutoML+import pandas as pd++train_dataset = {+ "id": ["0", "1"],+ "ner_tags": [+ ["B-ORG", "O", "B-MISC", "O", "O", "O", "B-MISC", "O", "O"],+ ["B-PER", "I-PER"],+ ],+ "tokens": [+ [+ "EU", "rejects", "German", "call", "to", "boycott", "British", "lamb", ".",+ ],+ ["Peter", "Blackburn"],+ ],+}+dev_dataset = {+ "id": ["0"],+ "ner_tags": [+ ["O"],+ ],+ "tokens": [+ ["1996-08-22"]+ ],+}+test_dataset = {+ "id": ["0"],+ "ner_tags": [+ ["O"],+ ],+ "tokens": [+ ['.']+ ],+}+custom_sent_keys = ["tokens"]+label_key = "ner_tags"++train_dataset = pd.DataFrame(train_dataset)+dev_dataset = pd.DataFrame(dev_dataset)+test_dataset = pd.DataFrame(test_dataset)++X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]+X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]+X_test = test_dataset[custom_sent_keys]++automl = AutoML()+automl_settings = {+ "time_budget": 10,+ "task": "token-classification",+ "fit_kwargs_by_estimator": {+ "transformer":+ {+ "output_dir": "data/output/"+ # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base+ }+ }, # setting the huggingface arguments: output directory+ "gpu_per_trial": 1, # set to 0 if no GPU is available+ "metric": "seqeval:overall_f1"+}++automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)+automl.predict(X_test)+```++The second is to define the id labels + a token [label list](https://microsoft.github.io/FLAML/docs/reference/nlp/huggingface/training_args):++```python+from flaml import AutoML+import pandas as pd++train_dataset = {+ "id": ["0", "1"],+ "ner_tags": [+ [3, 0, 7, 0, 0, 0, 7, 0, 0],+ [1, 2],+ ],+ "tokens": [+ [+ "EU", "rejects", "German", "call", "to", "boycott", "British", "lamb", ".",+ ],+ ["Peter", "Blackburn"],+ ],+ }+dev_dataset = {+ "id": ["0"],+ "ner_tags": [+ [0],+ ],+ "tokens": [+ ["1996-08-22"]+ ],+}+test_dataset = {+ "id": ["0"],+ "ner_tags": [+ [0],+ ],+ "tokens": [+ ['.']+ ],+}+custom_sent_keys = ["tokens"]+label_key = "ner_tags"++train_dataset = pd.DataFrame(train_dataset)+dev_dataset = pd.DataFrame(dev_dataset)+test_dataset = pd.DataFrame(test_dataset)++X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]+X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]+X_test = test_dataset[custom_sent_keys]++automl = AutoML()+automl_settings = {+ "time_budget": 10,+ "task": "token-classification",+ "fit_kwargs_by_estimator": {+ "transformer":+ {+ "output_dir": "data/output/",+ # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base+ "label_list": [ "O","B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC", "B-MISC", "I-MISC" ]+ }+ }, # setting the huggingface arguments: output directory+ "gpu_per_trial": 1, # set to 0 if no GPU is available+ "metric": "seqeval:overall_f1"+}++automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)+automl.predict(X_test)+```++#### Sample Output++```+[flaml.automl: 06-30 03:10:02] {2423} INFO - task = token-classification+[flaml.automl: 06-30 03:10:02] {2425} INFO - Data split method: stratified+[flaml.automl: 06-30 03:10:02] {2428} INFO - Evaluation method: holdout+[flaml.automl: 06-30 03:10:02] {2497} INFO - Minimizing error metric: seqeval:overall_f1+[flaml.automl: 06-30 03:10:02] {2637} INFO - List of ML learners in AutoML Run: ['transformer']+[flaml.automl: 06-30 03:10:02] {2929} INFO - iteration 0, current learner transformer+```+For tasks that are not currently supported, use `flaml.tune` for [customized tuning](Tune-HuggingFace).### Link to Jupyter notebook