A programming framework for agentic AI
fixing a bug in nlp/utils.py (#590)
Xueqing Liu79a24d06a934
diff --git a/flaml/nlp/utils.py b/flaml/nlp/utils.pyindex cd2e7a409..390b05c71 100644--- a/flaml/nlp/utils.py+++ b/flaml/nlp/utils.py@@ -84,10 +84,10 @@ def tokenize_and_align_labels(tokenized_inputs = tokenizer([list(examples[X_sent_key])],padding="max_length"- if hf_args.pad_to_max_length+ if hf_args and hf_args.pad_to_max_lengthelse False, # to be consistent with https://github.com/huggingface/transformers/blob/main/examples/pytorch/token-classification/run_ner.py#L394truncation=True,- max_length=hf_args.max_seq_length,+ max_length=hf_args.max_seq_length if hf_args else None,# We use this argument because the texts in our dataset are lists of words (with a label for each word).is_split_into_words=True,)