Skip to content

A programming framework for agentic AI

fix ner bug; refactor post processing of TransformersEstimator prediction (#615)

Xueqing Liu6108493e0b77
diff --git a/flaml/automl.py b/flaml/automl.py
index 37650d359..3b065504a 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -1108,10 +1108,21 @@ class AutoML(BaseEstimator):
from .data import DataTransformer
self._transformer = DataTransformer()
+
self._X_train_all, self._y_train_all = self._transformer.fit_transform(
X, y, self._state.task
)
self._label_transformer = self._transformer.label_transformer
+ if hasattr(self._label_transformer, "label_list"):
+ self._state.fit_kwargs.update(
+ {"label_list": self._label_transformer.label_list}
+ )
+ elif self._state.task == TOKENCLASSIFICATION:
+ if "label_list" not in self._state.fit_kwargs:
+ for each_fit_kwargs in self._state.fit_kwargs_by_estimator.values():
+ assert (
+ "label_list" in each_fit_kwargs
+ ), "For the token-classification task, you must either (1) pass token labels; or (2) pass id labels and the label list. Please refer to the documentation for more details: https://microsoft.github.io/FLAML/docs/Examples/AutoML-NLP#a-simple-token-classification-example"
self._sample_weight_full = self._state.fit_kwargs.get(
"sample_weight"
diff --git a/flaml/data.py b/flaml/data.py
index 379147ac0..28960a0e2 100644
--- a/flaml/data.py
+++ b/flaml/data.py
@@ -371,14 +371,20 @@ class DataTransformer:
)
self._drop = drop
if (
- (task in CLASSIFICATION or not pd.api.types.is_numeric_dtype(y))
+ task in CLASSIFICATION
+ or not pd.api.types.is_numeric_dtype(y)
and task not in NLG_TASKS
- and task != TOKENCLASSIFICATION
):
- from sklearn.preprocessing import LabelEncoder
+ if task != TOKENCLASSIFICATION:
+ from sklearn.preprocessing import LabelEncoder
- self.label_transformer = LabelEncoder()
+ self.label_transformer = LabelEncoder()
+ else:
+ from .nlp.utils import LabelEncoderforTokenClassification
+
+ self.label_transformer = LabelEncoderforTokenClassification()
y = self.label_transformer.fit_transform(y)
+
else:
self.label_transformer = None
self._task = task
diff --git a/flaml/ml.py b/flaml/ml.py
index 75d67f4be..49c0b0bf0 100644
--- a/flaml/ml.py
+++ b/flaml/ml.py
@@ -134,15 +134,22 @@ def get_estimator_class(task, estimator_name):
def metric_loss_score(
metric_name,
- y_predict,
- y_true,
+ y_processed_predict,
+ y_processed_true,
labels=None,
sample_weight=None,
groups=None,
):
+ # y_processed_predict and y_processed_true are processed id labels if the original were the token labels
+
if is_in_sklearn_metric_name_set(metric_name):
return sklearn_metric_loss_score(
- metric_name, y_predict, y_true, labels, sample_weight, groups
+ metric_name,
+ y_processed_predict,
+ y_processed_true,
+ labels,
+ sample_weight,
+ groups,
)
else:
"""
@@ -151,14 +158,22 @@ def metric_loss_score(
if metric_name == "spearmanr":
from scipy.stats import spearmanr
- y_true = y_true.to_list() if type(y_true) == pd.Series else list(y_true)
- score = spearmanr(list(y_predict), y_true)[0]
+ y_true = (
+ y_processed_true.to_list()
+ if isinstance(y_processed_true, pd.Series)
+ else list(y_processed_true)
+ )
+ score = spearmanr(list(y_processed_predict), y_true)[0]
metric_mode = "max"
elif metric_name == "pearsonr":
from scipy.stats import pearsonr
- y_true = y_true.to_list() if type(y_true) == pd.Series else list(y_true)
- score = pearsonr(list(y_predict), y_true)[0]
+ y_true = (
+ y_processed_true.to_list()
+ if type(y_processed_true) == pd.Series
+ else list(y_processed_true)
+ )
+ score = pearsonr(list(y_processed_predict), y_true)[0]
metric_mode = "max"
else:
try:
@@ -171,43 +186,31 @@ def metric_loss_score(
metric_mode = huggingface_metric_to_mode[datasets_metric_name]
if "rouge" in metric_name:
- score = metric.compute(predictions=y_predict, references=y_true)[
- metric_name
- ].mid.fmeasure
+ score = metric.compute(
+ predictions=y_processed_predict, references=y_processed_true
+ )[metric_name].mid.fmeasure
elif metric_name.startswith("seqeval"):
- label_len = len(labels)
- zip_pred_true = [
- [(p, lb) for (p, lb) in zip(prediction, label) if lb != -100]
- for (prediction, label) in zip(y_predict, y_true)
+ y_processed_true = [
+ [labels[tr] for tr in each_list]
+ for each_list in y_processed_true
]
- y_pred = [
- [
- labels[p] if 0 <= p < label_len else -1
- for (p, l) in each_list
- ]
- for each_list in zip_pred_true
- ] # To compute precision and recall, y_pred and y_true must be converted to string labels
- # (B-PER, I-PER, etc.), so that the category-based precision/recall (i.e., PER, LOC, etc.) scores can be computed
- y_true = [
- [labels[l] for (p, l) in each_list]
- for each_list in zip_pred_true
- ]
-
metric_submetric_names = metric_name.split(":")
- score = metric.compute(predictions=y_pred, references=y_true)[
+ score = metric.compute(
+ predictions=y_processed_predict, references=y_processed_true
+ )[
metric_submetric_names[1]
if len(metric_submetric_names) > 1
else "overall_accuracy"
]
else:
- score = metric.compute(predictions=y_predict, references=y_true)[
- metric_name
- ]
+ score = metric.compute(
+ predictions=y_processed_predict, references=y_processed_true
+ )[metric_name]
except ImportError:
- raise Exception(
+ raise ValueError(
metric_name
+ " is not an built-in sklearn metric and nlp is not installed. "
"Currently built-in sklearn metrics are: "
@@ -219,7 +222,7 @@ def metric_loss_score(
# If the metric is not found from huggingface dataset metric list (i.e., FileNotFoundError)
# ask the user to provide a custom metric
except FileNotFoundError:
- raise Exception(
+ raise ValueError(
metric_name
+ " is neither an sklearn metric nor a huggingface metric. "
"Currently built-in sklearn metrics are: "
@@ -265,14 +268,13 @@ def sklearn_metric_loss_score(
used to calculate the metric. E.g., 2d for log_loss and 1d
for others.
y_true: A 1d numpy array of the true labels.
- labels: A 1d numpy array of the unique labels.
+ labels: A list or an array of the unique labels.
sample_weight: A 1d numpy array of the sample weight.
groups: A 1d numpy array of the group labels.
Returns:
score: A float number of the loss, the lower the better.
"""
-
metric_name = metric_name.lower()
if "r2" == metric_name:
@@ -368,8 +370,14 @@ def _eval_estimator(
pred_start = time.time()
val_pred_y = get_y_pred(estimator, X_val, eval_metric, obj)
pred_time = (time.time() - pred_start) / X_val.shape[0]
+
val_loss = metric_loss_score(
- eval_metric, val_pred_y, y_val, labels, weight_val, groups_val
+ eval_metric,
+ y_processed_predict=val_pred_y,
+ y_processed_true=y_val,
+ labels=labels,
+ sample_weight=weight_val,
+ groups=groups_val,
)
metric_for_logging = {"pred_time": pred_time}
if log_training_metric:
@@ -588,6 +596,7 @@ def compute_estimator(
)
if isinstance(estimator, TransformersEstimator):
+ # TODO: move the partial function to nlp
fit_kwargs["metric"] = eval_metric
fit_kwargs["X_val"] = X_val
fit_kwargs["y_val"] = y_val
diff --git a/flaml/model.py b/flaml/model.py
index f73187898..f6318306c 100644
--- a/flaml/model.py
+++ b/flaml/model.py
@@ -522,14 +522,12 @@ class TransformersEstimator(BaseEstimator):
@property
def num_labels(self):
- from .data import SEQCLASSIFICATION, SEQREGRESSION, TOKENCLASSIFICATION
-
if self._task == SEQREGRESSION:
return 1
elif self._task == SEQCLASSIFICATION:
return len(set(self._y_train))
elif self._task == TOKENCLASSIFICATION:
- return len(set([a for b in self._y_train.tolist() for a in b]))
+ return len(self._training_args.label_list)
else:
return None
@@ -549,9 +547,9 @@ class TransformersEstimator(BaseEstimator):
return AutoTokenizer.from_pretrained(
self._training_args.model_path,
use_fast=True,
- add_prefix_space=True
- if "roberta" in self._training_args.model_path
- else False, # If roberta model, must set add_prefix_space to True to avoid the assertion error at
+ add_prefix_space="roberta" in self._training_args.model_path
+ and not getattr(self, "_pred_flag", False),
+ # If roberta model and the call is from .fit instead of .predict (when the model_path is updated to the checkpoint name instead), must set add_prefix_space to True to avoid the assertion error at
# https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249
)
@@ -734,39 +732,30 @@ class TransformersEstimator(BaseEstimator):
return best_ckpt
def _compute_metrics_by_dataset_name(self, eval_pred):
+ # TODO: call self._metric(eval_pred, self)
if isinstance(self._metric, str):
from .ml import metric_loss_score
- from .nlp.utils import postprocess_text
-
- predictions, labels = eval_pred
- if self._task in NLG_TASKS:
- if isinstance(predictions, tuple):
- predictions = np.argmax(predictions[0], axis=2)
- decoded_preds = self.tokenizer.batch_decode(
- predictions, skip_special_tokens=True
- )
- labels = np.where(labels != -100, labels, self.tokenizer.pad_token_id)
- decoded_labels = self.tokenizer.batch_decode(
- labels, skip_special_tokens=True
- )
- predictions, labels = postprocess_text(decoded_preds, decoded_labels)
- else:
- predictions = (
- np.squeeze(predictions)
- if self._task == SEQREGRESSION
- else np.argmax(predictions, axis=2)
- if self._task == TOKENCLASSIFICATION
- else np.argmax(predictions, axis=1)
- )
+ from .nlp.utils import postprocess_prediction_and_true
+
+ predictions, y_true = eval_pred
+ # postprocess the matrix prediction and ground truth into user readable format, e.g., for summarization, decode into text
+ processed_predictions, processed_y_true = postprocess_prediction_and_true(
+ task=self._task,
+ y_pred=predictions,
+ tokenizer=self.tokenizer,
+ hf_args=self._training_args,
+ y_true=y_true,
+ )
metric_dict = {
"automl_metric": metric_loss_score(
metric_name=self._metric,
- y_predict=predictions,
- y_true=labels,
+ y_processed_predict=processed_predictions,
+ y_processed_true=processed_y_true,
labels=self._training_args.label_list,
)
}
else:
+ # TODO: debug to see how custom metric can take both tokenized (here) and untokenized input (ml.py)
loss, metric_dict = self._metric(
X_test=self._X_val,
y_test=self._y_val,
@@ -786,6 +775,7 @@ class TransformersEstimator(BaseEstimator):
Need to reinit training_args because of a bug in deepspeed: if not reinit, the deepspeed config will be inconsistent
with HF config https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py#L947
"""
+ self._pred_flag = True
training_args = self._TrainingArguments(
local_rank=-1, model_path=self._checkpoint_path, fp16=self.fp16
)
@@ -837,6 +827,7 @@ class TransformersEstimator(BaseEstimator):
def predict(self, X, **pred_kwargs):
import transformers
from datasets import Dataset
+ from .nlp.utils import postprocess_prediction_and_true
transformers.logging.set_verbosity_error()
@@ -856,20 +847,14 @@ class TransformersEstimator(BaseEstimator):
test_dataset,
metric_key_prefix="predict",
)
-
- if self._task == SEQCLASSIFICATION:
- return np.argmax(predictions.predictions, axis=1)
- elif self._task == SEQREGRESSION:
- return predictions.predictions.reshape((len(predictions.predictions),))
- elif self._task == TOKENCLASSIFICATION:
- return np.argmax(predictions.predictions, axis=2)
- elif self._task == SUMMARIZATION:
- decoded_preds = self.tokenizer.batch_decode(
- predictions.predictions, skip_special_tokens=True
- )
- return decoded_preds
- elif self._task == MULTICHOICECLASSIFICATION:
- return np.argmax(predictions.predictions, axis=1)
+ post_y_pred, _ = postprocess_prediction_and_true(
+ task=self._task,
+ y_pred=predictions.predictions,
+ tokenizer=self.tokenizer,
+ hf_args=self._training_args,
+ X=X,
+ )
+ return post_y_pred
def config2params(self, config: dict) -> dict:
params = super().config2params(config)
diff --git a/flaml/nlp/huggingface/training_args.py b/flaml/nlp/huggingface/training_args.py
index e65b3bb0a..8790d2a5f 100644
--- a/flaml/nlp/huggingface/training_args.py
+++ b/flaml/nlp/huggingface/training_args.py
@@ -30,8 +30,10 @@ class TrainingArgumentsForAuto(TrainingArguments):
ckpt_per_epoch (int, optional, defaults to 1): An integer, the number of checkpoints per epoch.
per_device_eval_batch_size (int, optional, defaults to 1): An integer, the per gpu evaluation batch size.
label_list (List[str], optional, defaults to None): A list of string, the string list of the label names.
- When the task is sequence labeling/token classification, need to set the label_list (e.g., B-PER, I-PER, B-LOC)
- to obtain the correct evaluation metric. See the example in test/nlp/test_autohf_tokenclassification.py.
+ When the task is sequence labeling/token classification, there are two formats of the labels:
+ (1) The token labels, i.e., [B-PER, I-PER, B-LOC]; (2) Id labels. For (2), need to pass the label_list (e.g., [B-PER, I-PER, B-LOC])
+ to convert the Id to token labels when computing the metric with metric_loss_score.
+ See the example in [a simple token classification example](../../../Examples/AutoML-NLP#a-simple-token-classification-example).
"""
task: str = field(default="seq-classification")
@@ -49,6 +51,13 @@ class TrainingArgumentsForAuto(TrainingArguments):
max_seq_length: int = field(default=128, metadata={"help": "max seq length"})
+ label_all_tokens: bool = field(
+ default=False,
+ metadata={
+ "help": "For NER task, whether to set the extra tokenized labels to the same label (instead of -100)"
+ },
+ )
+
pad_to_max_length: bool = field(
default=False,
metadata={
diff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.py
index 390b05c71..5f0d1464e 100644
--- a/flaml/nlp/utils.py
+++ b/flaml/nlp/utils.py
@@ -11,6 +11,8 @@ from ..data import (
NLG_TASKS,
)
+import pandas as pd
+
def load_default_huggingface_metric_for_task(task):
@@ -54,28 +56,30 @@ def tokenize_seq2seq(X, Y, tokenizer, task=None, hf_args=None):
hf_args=hf_args,
prefix_str="summarize: ",
)
- labels = None
+ model_outputs = None
if Y is not None:
- labels = tokenize_onedataframe(
+ model_outputs = tokenize_onedataframe(
Y.to_frame(),
tokenizer=tokenizer,
task=task,
hf_args=hf_args,
prefix_str="",
)
- labels["label"] = [
+ model_outputs["label"] = [
[(each_l if each_l != tokenizer.pad_token_id else -100) for each_l in label]
- for label in labels["input_ids"]
+ for label in model_outputs["input_ids"]
]
- labels = labels.drop(
+ model_outputs = model_outputs.drop(
columns=["attention_mask", "input_ids", "decoder_input_ids"]
)
- return model_inputs, labels
+ return model_inputs, model_outputs
def tokenize_and_align_labels(
examples,
tokenizer,
+ label_to_id,
+ b_to_i_label,
hf_args=None,
X_sent_key=None,
Y_sent_key=None,
@@ -94,26 +98,22 @@ def tokenize_and_align_labels(
if Y_sent_key is not None:
previous_word_idx = None
label_ids = []
- import numbers
-
for word_idx in tokenized_inputs.word_ids(batch_index=0):
- # Special tokens have a word id that is None. We set the label to -100 so they are automatically
- # ignored in the loss function.
if word_idx is None:
label_ids.append(-100)
- # We set the label for the first token of each word.
elif word_idx != previous_word_idx:
- if isinstance(examples[Y_sent_key][word_idx], numbers.Number):
- label_ids.append(examples[Y_sent_key][word_idx])
- # else:
- # label_ids.append(label_to_id[label[word_idx]])
+ label_ids.append(label_to_id[examples[Y_sent_key][word_idx]])
# For the other tokens in a word, we set the label to either the current label or -100, depending on
# the label_all_tokens flag.
else:
- if isinstance(examples[Y_sent_key][word_idx], numbers.Number):
- label_ids.append(examples[Y_sent_key][word_idx])
- # else:
- # label_ids.append(b_to_i_label[label_to_id[label[word_idx]]])
+ # Use the label_all_tokens to control whether to copy the label to all subtokens or to pad the additional tokens as -100
+ if hf_args.label_all_tokens:
+ # If the B- word is converted into multiple subtokens, map the additional subtokens to I-
+ label_ids.append(
+ b_to_i_label[label_to_id[examples[Y_sent_key][word_idx]]]
+ )
+ else:
+ label_ids.append(-100)
previous_word_idx = word_idx
tokenized_inputs["labels"] = label_ids
tmp_column_names = sorted(tokenized_inputs.keys())
@@ -128,7 +128,15 @@ def tokenize_and_align_labels(
def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):
- import pandas as pd
+
+ # If the label_all_tokens flag is True, prepare two dicts label_to_id and b_to_i_label to convert the B- labels to I- labels
+ label_to_id = {i: i for i in range(len(hf_args.label_list))}
+ b_to_i_label = []
+ for idx, label in enumerate(hf_args.label_list):
+ if label.startswith("B-") and label.replace("B-", "I-") in hf_args.label_list:
+ b_to_i_label.append(hf_args.label_list.index(label.replace("B-", "I-")))
+ else:
+ b_to_i_label.append(idx)
if Y is not None:
X_and_Y = pd.concat([X, Y.to_frame()], axis=1)
@@ -141,6 +149,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):
X_sent_key=X_key,
Y_sent_key=Y_key,
return_column_name=True,
+ label_to_id=label_to_id,
+ b_to_i_label=b_to_i_label,
)
X_and_Y_tokenized = X_and_Y.apply(
lambda x: tokenize_and_align_labels(
@@ -149,6 +159,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):
hf_args=hf_args,
X_sent_key=X_key,
Y_sent_key=Y_key,
+ label_to_id=label_to_id,
+ b_to_i_label=b_to_i_label,
),
axis=1,
result_type="expand",
@@ -170,6 +182,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):
X_sent_key=X_key,
Y_sent_key=None,
return_column_name=True,
+ label_to_id=label_to_id,
+ b_to_i_label=b_to_i_label,
)
d = X.apply(
@@ -179,6 +193,8 @@ def tokenize_text_tokclassification(X, Y, tokenizer, hf_args=None):
hf_args=hf_args,
X_sent_key=X_key,
Y_sent_key=None,
+ label_to_id=label_to_id,
+ b_to_i_label=b_to_i_label,
),
axis=1,
result_type="expand",
@@ -197,7 +213,6 @@ def tokenize_onedataframe(
hf_args=None,
prefix_str=None,
):
- import pandas
with tokenizer.as_target_tokenizer():
_, tokenized_column_names = tokenize_row(
@@ -219,25 +234,11 @@ def tokenize_onedataframe(
axis=1,
result_type="expand",
)
- X_tokenized = pandas.DataFrame(columns=tokenized_column_names)
+ X_tokenized = pd.DataFrame(columns=tokenized_column_names)
X_tokenized[tokenized_column_names] = d
return X_tokenized
-def postprocess_text(preds, labels):
- import nltk
-
- nltk.download("punkt")
- preds = [pred.strip() for pred in preds]
- labels = [label.strip() for label in labels]
-
- # rougeLSum expects newline after each sentence
- preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds]
- labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels]
-
- return preds, labels
-
-
def tokenize_row(
this_row,
tokenizer,
@@ -267,7 +268,6 @@ def tokenize_row(
def tokenize_text_multiplechoice(X, tokenizer, hf_args=None):
- import pandas
t = X[["sent1", "sent2", "ending0", "ending1", "ending2", "ending3"]]
_, tokenized_column_names = tokenize_swag(
@@ -282,7 +282,7 @@ def tokenize_text_multiplechoice(X, tokenizer, hf_args=None):
result_type="expand",
)
- X_tokenized = pandas.DataFrame(columns=tokenized_column_names)
+ X_tokenized = pd.DataFrame(columns=tokenized_column_names)
X_tokenized[tokenized_column_names] = d
output = X_tokenized.join(X)
return output, None
@@ -475,3 +475,98 @@ def load_model(checkpoint_path, task, num_labels=None):
model_config = _set_model_config(checkpoint_path)
this_model = get_this_model(checkpoint_path, task, model_config)
return this_model
+
+
+def postprocess_prediction_and_true(
+ task, y_pred, tokenizer, hf_args, y_true=None, X=None
+):
+ # postprocess the matrix prediction y_pred and ground truth y_true into user readable format, e.g., for summarization, decode into text
+ if task == SEQCLASSIFICATION:
+ return np.argmax(y_pred, axis=1), y_true
+ elif task == SEQREGRESSION:
+ return np.squeeze(y_pred), y_true # predictions.reshape((len(predictions),))
+ elif task == TOKENCLASSIFICATION:
+ assert (y_true is not None) or (
+ X is not None
+ ), "One of y_true and X must not be None"
+ ## If y_true is not None, we use y_true to remove the -100 in the prediction (postprocessing), and return the postprocessed y_true and prediction
+ # If y_true is None, we use X to compute y_is_pad (i.e., whether y_true is -100 in that position), and use y_is_pad to remove the -100 in the prediction, and return the postprocessed prediction (not the y_true)
+ y_predict = pd.Series(np.argmax(y_pred, axis=2).tolist())
+ if y_true is None:
+ _, y_is_pad = tokenize_text(
+ X,
+ y_predict,
+ task=task,
+ hf_args=hf_args,
+ tokenizer=tokenizer,
+ )
+ else:
+ y_is_pad = y_true
+ label_len = len(hf_args.label_list)
+ zip_pred_ispad = [
+ [(p, ispd) for (p, ispd) in zip(each_pred, each_is_pad) if ispd != -100]
+ for (each_pred, each_is_pad) in zip(y_predict, y_is_pad)
+ ]
+ y_pred_label = [
+ [
+ hf_args.label_list[p] if 0 <= p < label_len else -1
+ for (p, ispd) in each_list
+ ]
+ for each_list in zip_pred_ispad
+ ] # To compute precision and recall, y_pred and y_true must be converted to string labels
+ # (B-PER, I-PER, etc.), so that the category-based precision/recall (i.e., PER, LOC, etc.) scores can be computed
+ if y_true is not None:
+ y_true_label = [
+ [tr for (p, tr) in each_list] for each_list in zip_pred_ispad
+ ]
+ else:
+ y_true_label = None
+ return y_pred_label, y_true_label
+ elif task == SUMMARIZATION:
+ if isinstance(y_pred, tuple):
+ y_pred = np.argmax(y_pred[0], axis=2)
+ decoded_preds = tokenizer.batch_decode(y_pred, skip_special_tokens=True)
+
+ import nltk
+
+ nltk.download("punkt")
+ decoded_preds = [pred.strip() for pred in decoded_preds]
+ decoded_preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in decoded_preds]
+
+ if y_true is not None:
+ y_true_labels = np.where(y_true != -100, y_true, tokenizer.pad_token_id)
+ decoded_y_true_labels = tokenizer.batch_decode(
+ y_true_labels, skip_special_tokens=True
+ )
+ decoded_y_true_labels = [label.strip() for label in decoded_y_true_labels]
+ decoded_y_true_labels = [
+ "\n".join(nltk.sent_tokenize(label)) for label in decoded_y_true_labels
+ ]
+ else:
+ decoded_y_true_labels = None
+
+ return decoded_preds, decoded_y_true_labels
+ elif task == MULTICHOICECLASSIFICATION:
+ return np.argmax(y_pred, axis=1), y_true
+
+
+class LabelEncoderforTokenClassification:
+ def fit_transform(self, y):
+ # if the labels are tokens, convert them to ids
+ if any(isinstance(id, str) for id in y[0]):
+ self.label_list = sorted(list(set().union(*y)))
+ self._tokenlabel_to_id = {
+ self.label_list[id]: id for id in range(len(self.label_list))
+ }
+ y = y.apply(lambda sent: [self._tokenlabel_to_id[token] for token in sent])
+ # if the labels are not tokens, they must be ids
+ else:
+ assert all(
+ isinstance(id, int) for id in y[0]
+ ), "The labels must either be tokens or ids"
+ return y
+
+ def transform(self, y):
+ if hasattr(self, "_tokenlabel_to_id"):
+ y = y.apply(lambda sent: [self._tokenlabel_to_id[token] for token in sent])
+ return y
diff --git a/test/nlp/test_autohf_custom_metric.py b/test/nlp/test_autohf_custom_metric.py
index 3b8e0a360..ee5317d15 100644
--- a/test/nlp/test_autohf_custom_metric.py
+++ b/test/nlp/test_autohf_custom_metric.py
@@ -25,10 +25,10 @@ def custom_metric(
else:
trainer = estimator._trainer
if y_test is not None:
- X_test, _ = estimator._preprocess(X_test)
+ X_test = estimator._preprocess(X_test)
eval_dataset = Dataset.from_pandas(TransformersEstimator._join(X_test, y_test))
else:
- X_test, _ = estimator._preprocess(X_test)
+ X_test = estimator._preprocess(X_test)
eval_dataset = Dataset.from_pandas(X_test)
estimator_metric_backup = estimator._metric
diff --git a/test/nlp/test_autohf_summarization.py b/test/nlp/test_autohf_summarization.py
index ac05a7f2f..e52a8754b 100644
--- a/test/nlp/test_autohf_summarization.py
+++ b/test/nlp/test_autohf_summarization.py
@@ -6,6 +6,7 @@ from utils import get_toy_data_summarization, get_automl_settings
@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os")
def test_summarization():
+ # TODO: manual test for how effective postprocess_seq2seq_prediction_label is
from flaml import AutoML
X_train, y_train, X_val, y_val, X_test = get_toy_data_summarization()
diff --git a/test/nlp/test_autohf_tokenclassification.py b/test/nlp/test_autohf_tokenclassification.py
index 2a60617e9..cb1c40e9d 100644
--- a/test/nlp/test_autohf_tokenclassification.py
+++ b/test/nlp/test_autohf_tokenclassification.py
@@ -1,17 +1,21 @@
import sys
import pytest
import requests
-from utils import get_toy_data_tokenclassification, get_automl_settings
+from utils import (
+ get_toy_data_tokenclassification_idlabel,
+ get_toy_data_tokenclassification_tokenlabel,
+ get_automl_settings,
+)
@pytest.mark.skipif(
sys.platform == "darwin" or sys.version < "3.7",
reason="do not run on mac os or py<3.7",
)
-def test_tokenclassification():
+def test_tokenclassification_idlabel():
from flaml import AutoML
- X_train, y_train, X_val, y_val = get_toy_data_tokenclassification()
+ X_train, y_train, X_val, y_val = get_toy_data_tokenclassification_idlabel()
automl = AutoML()
automl_settings = get_automl_settings()
@@ -42,6 +46,66 @@ def test_tokenclassification():
except requests.exceptions.HTTPError:
return
+ # perf test
+ import json
+
+ with open("seqclass.log", "r") as fin:
+ for line in fin:
+ each_log = json.loads(line.strip("\n"))
+ if "validation_loss" in each_log:
+ val_loss = each_log["validation_loss"]
+ min_inter_result = min(
+ each_dict.get("eval_automl_metric", sys.maxsize)
+ for each_dict in each_log["logged_metric"]["intermediate_results"]
+ )
+
+ if min_inter_result != sys.maxsize:
+ assert val_loss == min_inter_result
+
+
+ sys.platform == "darwin" or sys.version < "3.7",
+ reason="do not run on mac os or py<3.7",
+)
+def test_tokenclassification_tokenlabel():
+ from flaml import AutoML
+
+ X_train, y_train, X_val, y_val = get_toy_data_tokenclassification_tokenlabel()
+ automl = AutoML()
+
+ automl_settings = get_automl_settings()
+ automl_settings["task"] = "token-classification"
+ automl_settings[
+ "metric"
+ ] = "seqeval:overall_f1" # evaluating based on the overall_f1 of seqeval
+
+ try:
+ automl.fit(
+ X_train=X_train,
+ y_train=y_train,
+ X_val=X_val,
+ y_val=y_val,
+ **automl_settings
+ )
+ except requests.exceptions.HTTPError:
+ return
+
+ # perf test
+ import json
+
+ with open("seqclass.log", "r") as fin:
+ for line in fin:
+ each_log = json.loads(line.strip("\n"))
+ if "validation_loss" in each_log:
+ val_loss = each_log["validation_loss"]
+ min_inter_result = min(
+ each_dict.get("eval_automl_metric", sys.maxsize)
+ for each_dict in each_log["logged_metric"]["intermediate_results"]
+ )
+
+ if min_inter_result != sys.maxsize:
+ assert val_loss == min_inter_result
+
if __name__ == "__main__":
- test_tokenclassification()
+ test_tokenclassification_idlabel()
diff --git a/test/nlp/utils.py b/test/nlp/utils.py
index 8d07bada1..af512c90e 100644
--- a/test/nlp/utils.py
+++ b/test/nlp/utils.py
@@ -406,7 +406,8 @@ def get_toy_data_summarization():
return X_train, y_train, X_val, y_val, X_test
-def get_toy_data_tokenclassification():
+def get_toy_data_tokenclassification_idlabel():
+ # test token classification when the labels are ids
train_data = {
"chunk_tags": [
[11, 21, 11, 12, 21, 22, 11, 12, 0],
@@ -1116,6 +1117,391 @@ def get_toy_data_tokenclassification():
return X_train, y_train, X_val, y_val
+def get_toy_data_tokenclassification_tokenlabel():
+ # test token classification when the labels are tokens
+ train_data = {
+ "id": ["0", "1", "2", "3"],
+ "ner_tags": [
+ ["B-ORG", "O", "B-MISC", "O", "O", "O", "B-MISC", "O", "O"],
+ ["B-PER", "I-PER"],
+ ["B-LOC", "O"],
+ [
+ "O",
+ "B-ORG",
+ "I-ORG",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "B-MISC",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "B-MISC",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ ],
+ ],
+ "tokens": [
+ [
+ "EU",
+ "rejects",
+ "German",
+ "call",
+ "to",
+ "boycott",
+ "British",
+ "lamb",
+ ".",
+ ],
+ ["Peter", "Blackburn"],
+ ["BRUSSELS", "1996-08-22"],
+ [
+ "The",
+ "European",
+ "Commission",
+ "said",
+ "on",
+ "Thursday",
+ "it",
+ "disagreed",
+ "with",
+ "German",
+ "advice",
+ "to",
+ "consumers",
+ "to",
+ "shun",
+ "British",
+ "lamb",
+ "until",
+ "scientists",
+ "determine",
+ "whether",
+ "mad",
+ "cow",
+ "disease",
+ "can",
+ "be",
+ "transmitted",
+ "to",
+ "sheep",
+ ".",
+ ],
+ ],
+ }
+
+ dev_data = {
+ "id": ["4", "5", "6", "7"],
+ "ner_tags": [
+ [
+ "B-LOC",
+ "O",
+ "O",
+ "O",
+ "O",
+ "B-ORG",
+ "I-ORG",
+ "O",
+ "O",
+ "O",
+ "B-PER",
+ "I-PER",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "B-LOC",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ ],
+ [
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "B-ORG",
+ "O",
+ "O",
+ "O",
+ "B-PER",
+ "I-PER",
+ "I-PER",
+ "I-PER",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ ],
+ [
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "B-ORG",
+ "I-ORG",
+ "O",
+ ],
+ [
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "B-ORG",
+ "O",
+ "O",
+ "B-PER",
+ "I-PER",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ "O",
+ ],
+ ],
+ "tokens": [
+ [
+ "Germany",
+ "'s",
+ "representative",
+ "to",
+ "the",
+ "European",
+ "Union",
+ "'s",
+ "veterinary",
+ "committee",
+ "Werner",
+ "Zwingmann",
+ "said",
+ "on",
+ "Wednesday",
+ "consumers",
+ "should",
+ "buy",
+ "sheepmeat",
+ "from",
+ "countries",
+ "other",
+ "than",
+ "Britain",
+ "until",
+ "the",
+ "scientific",
+ "advice",
+ "was",
+ "clearer",
+ ".",
+ ],
+ [
+ '"',
+ "We",
+ "do",
+ "n't",
+ "support",
+ "any",
+ "such",
+ "recommendation",
+ "because",
+ "we",
+ "do",
+ "n't",
+ "see",
+ "any",
+ "grounds",
+ "for",
+ "it",
+ ",",
+ '"',
+ "the",
+ "Commission",
+ "'s",
+ "chief",
+ "spokesman",
+ "Nikolaus",
+ "van",
+ "der",
+ "Pas",
+ "told",
+ "a",
+ "news",
+ "briefing",
+ ".",
+ ],
+ [
+ "He",
+ "said",
+ "further",
+ "scientific",
+ "study",
+ "was",
+ "required",
+ "and",
+ "if",
+ "it",
+ "was",
+ "found",
+ "that",
+ "action",
+ "was",
+ "needed",
+ "it",
+ "should",
+ "be",
+ "taken",
+ "by",
+ "the",
+ "European",
+ "Union",
+ ".",
+ ],
+ [
+ "He",
+ "said",
+ "a",
+ "proposal",
+ "last",
+ "month",
+ "by",
+ "EU",
+ "Farm",
+ "Commissioner",
+ "Franz",
+ "Fischler",
+ "to",
+ "ban",
+ "sheep",
+ "brains",
+ ",",
+ "spleens",
+ "and",
+ "spinal",
+ "cords",
+ "from",
+ "the",
+ "human",
+ "and",
+ "animal",
+ "food",
+ "chains",
+ "was",
+ "a",
+ "highly",
+ "specific",
+ "and",
+ "precautionary",
+ "move",
+ "to",
+ "protect",
+ "human",
+ "health",
+ ".",
+ ],
+ ],
+ }
+ train_dataset = pd.DataFrame(train_data)
+ dev_dataset = pd.DataFrame(dev_data)
+
+ custom_sent_keys = ["tokens"]
+ label_key = "ner_tags"
+
+ X_train = train_dataset[custom_sent_keys]
+ y_train = train_dataset[label_key]
+
+ X_val = dev_dataset[custom_sent_keys]
+ y_val = dev_dataset[label_key]
+ return X_train, y_train, X_val, y_val
+
+
def get_automl_settings(estimator_name="transformer"):
automl_settings = {
diff --git a/website/docs/Examples/AutoML-NLP.md b/website/docs/Examples/AutoML-NLP.md
index 8976345cd..df3b971aa 100644
--- a/website/docs/Examples/AutoML-NLP.md
+++ b/website/docs/Examples/AutoML-NLP.md
@@ -26,7 +26,7 @@ automl = AutoML()
automl_settings = {
"time_budget": 100,
"task": "seq-classification",
- "fit_kwargs_by_estimator": {
+ "fit_kwargs_by_estimator": {
"transformer":
{
"output_dir": "data/output/" # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base
@@ -87,7 +87,7 @@ automl_settings["fit_kwargs_by_estimator"] = { # setting the huggingface argume
"model_path": "google/electra-small-discriminator", # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base
"output_dir": "data/output/", # setting the output directory
"ckpt_per_epoch": 5, # setting the number of checkpoints per epoch
- "fp16": False,
+ "fp16": False,
} # setting whether to use FP16
}
automl.fit(
@@ -139,7 +139,7 @@ automl_settings["fit_kwargs_by_estimator"] = { # setting the huggingface ar
"model_path": "t5-small", # if model_path is not set, the default model is t5-small: https://huggingface.co/t5-small
"output_dir": "data/output/", # setting the output directory
"ckpt_per_epoch": 5, # setting the number of checkpoints per epoch
- "fp16": False,
+ "fp16": False,
} # setting whether to use FP16
}
automl.fit(
@@ -214,6 +214,154 @@ Model config T5Config {
}
```
+### A simple token classification example
+
+There are two ways to define the label for a token classification task. The first is to define the token labels:
+
+```python
+from flaml import AutoML
+import pandas as pd
+
+train_dataset = {
+ "id": ["0", "1"],
+ "ner_tags": [
+ ["B-ORG", "O", "B-MISC", "O", "O", "O", "B-MISC", "O", "O"],
+ ["B-PER", "I-PER"],
+ ],
+ "tokens": [
+ [
+ "EU", "rejects", "German", "call", "to", "boycott", "British", "lamb", ".",
+ ],
+ ["Peter", "Blackburn"],
+ ],
+}
+dev_dataset = {
+ "id": ["0"],
+ "ner_tags": [
+ ["O"],
+ ],
+ "tokens": [
+ ["1996-08-22"]
+ ],
+}
+test_dataset = {
+ "id": ["0"],
+ "ner_tags": [
+ ["O"],
+ ],
+ "tokens": [
+ ['.']
+ ],
+}
+custom_sent_keys = ["tokens"]
+label_key = "ner_tags"
+
+train_dataset = pd.DataFrame(train_dataset)
+dev_dataset = pd.DataFrame(dev_dataset)
+test_dataset = pd.DataFrame(test_dataset)
+
+X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]
+X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]
+X_test = test_dataset[custom_sent_keys]
+
+automl = AutoML()
+automl_settings = {
+ "time_budget": 10,
+ "task": "token-classification",
+ "fit_kwargs_by_estimator": {
+ "transformer":
+ {
+ "output_dir": "data/output/"
+ # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base
+ }
+ }, # setting the huggingface arguments: output directory
+ "gpu_per_trial": 1, # set to 0 if no GPU is available
+ "metric": "seqeval:overall_f1"
+}
+
+automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)
+automl.predict(X_test)
+```
+
+The second is to define the id labels + a token [label list](https://microsoft.github.io/FLAML/docs/reference/nlp/huggingface/training_args):
+
+```python
+from flaml import AutoML
+import pandas as pd
+
+train_dataset = {
+ "id": ["0", "1"],
+ "ner_tags": [
+ [3, 0, 7, 0, 0, 0, 7, 0, 0],
+ [1, 2],
+ ],
+ "tokens": [
+ [
+ "EU", "rejects", "German", "call", "to", "boycott", "British", "lamb", ".",
+ ],
+ ["Peter", "Blackburn"],
+ ],
+ }
+dev_dataset = {
+ "id": ["0"],
+ "ner_tags": [
+ [0],
+ ],
+ "tokens": [
+ ["1996-08-22"]
+ ],
+}
+test_dataset = {
+ "id": ["0"],
+ "ner_tags": [
+ [0],
+ ],
+ "tokens": [
+ ['.']
+ ],
+}
+custom_sent_keys = ["tokens"]
+label_key = "ner_tags"
+
+train_dataset = pd.DataFrame(train_dataset)
+dev_dataset = pd.DataFrame(dev_dataset)
+test_dataset = pd.DataFrame(test_dataset)
+
+X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]
+X_val, y_val = dev_dataset[custom_sent_keys], dev_dataset[label_key]
+X_test = test_dataset[custom_sent_keys]
+
+automl = AutoML()
+automl_settings = {
+ "time_budget": 10,
+ "task": "token-classification",
+ "fit_kwargs_by_estimator": {
+ "transformer":
+ {
+ "output_dir": "data/output/",
+ # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base
+ "label_list": [ "O","B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC", "B-MISC", "I-MISC" ]
+ }
+ }, # setting the huggingface arguments: output directory
+ "gpu_per_trial": 1, # set to 0 if no GPU is available
+ "metric": "seqeval:overall_f1"
+}
+
+automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)
+automl.predict(X_test)
+```
+
+#### Sample Output
+
+```
+[flaml.automl: 06-30 03:10:02] {2423} INFO - task = token-classification
+[flaml.automl: 06-30 03:10:02] {2425} INFO - Data split method: stratified
+[flaml.automl: 06-30 03:10:02] {2428} INFO - Evaluation method: holdout
+[flaml.automl: 06-30 03:10:02] {2497} INFO - Minimizing error metric: seqeval:overall_f1
+[flaml.automl: 06-30 03:10:02] {2637} INFO - List of ML learners in AutoML Run: ['transformer']
+[flaml.automl: 06-30 03:10:02] {2929} INFO - iteration 0, current learner transformer
+```
+
For tasks that are not currently supported, use `flaml.tune` for [customized tuning](Tune-HuggingFace).
### Link to Jupyter notebook