A programming framework for agentic AI
fixing roberta add_prefix_space bug (#546)
Xueqing Liu2ca9e41e4bea
diff --git a/flaml/model.py b/flaml/model.pyindex 0ecac416f..78d423f4c 100644--- a/flaml/model.py+++ b/flaml/model.py@@ -542,7 +542,13 @@ class TransformersEstimator(BaseEstimator):)else:return AutoTokenizer.from_pretrained(- self._training_args.model_path, use_fast=True+ self._training_args.model_path,+ use_fast=True,+ add_prefix_space=True+ if "roberta" in self._training_args.model_path+ else False, # If roberta model, must set add_prefix_space to True to avoid the assertion error at++ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249)@property