Skip to content

A programming framework for agentic AI

fixing roberta add_prefix_space bug (#546)

Xueqing Liu2ca9e41e4bea
diff --git a/flaml/model.py b/flaml/model.py
index 0ecac416f..78d423f4c 100644
--- a/flaml/model.py
+++ b/flaml/model.py
@@ -542,7 +542,13 @@ class TransformersEstimator(BaseEstimator):
)
else:
return AutoTokenizer.from_pretrained(
- self._training_args.model_path, use_fast=True
+ self._training_args.model_path,
+ use_fast=True,
+ add_prefix_space=True
+ if "roberta" in self._training_args.model_path
+ else False, # If roberta model, must set add_prefix_space to True to avoid the assertion error at
+
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/tokenization_roberta_fast.py#L249
)
@property