LLM inference in C/C++
C++ ggml
gguf-py : ByteLevel processing defaults bos/eos to False (#29422)
Sigbjørn Skjæretb248f4a3c163
diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.pyindex b5eaa71dd2..31036bbe5f 100644--- a/gguf-py/gguf/vocab.py+++ b/gguf-py/gguf/vocab.py@@ -234,7 +234,11 @@ class SpecialVocab:tokenizer_config['eos_token'] = special_eos = special_sepif post_processor := tokenizer.get('post_processor'):for processor in post_processor.get('processors', [post_processor]):- if processor.get('type') == 'RobertaProcessing':+ processor_type = processor.get('type')+ if processor_type == 'ByteLevel':+ self.add_special_token.setdefault('bos', False)+ self.add_special_token.setdefault('eos', False)+ elif processor_type == 'RobertaProcessing':self.add_special_token['bos'] = Trueself.add_special_token['eos'] = Trueself.add_special_token['sep'] = True@@ -247,7 +251,7 @@ class SpecialVocab:continue# Crude parsing of TemplateProcessing to determine if BOS/SEP/EOS should be added# Only works with simple templates, **will** get it wrong on unusual sequences- if processor.get('type') == 'TemplateProcessing':+ elif processor_type == 'TemplateProcessing':tmpl_single = processor.get('single', [])tmpl_pair = processor.get('pair', [])special_first = None@@ -339,7 +343,7 @@ class SpecialVocab:if typ not in self.add_special_token:self.add_special_token[typ] = add_entryelif self.add_special_token[typ] != add_entry:- logger.warning(f'Mismatch between tokenizer_config add_{typ}_token({add_entry}) and TemplateProcessing<{typ}>({self.add_special_token[typ]}) - opting for the latter')+ logger.warning(f'Mismatch between tokenizer_config add_{typ}_token({add_entry}) and tokenizer post_processor<{typ}>({self.add_special_token[typ]}) - opting for the latter')entry = tokenizer_config.get(f'{typ}_token')if isinstance(entry, str):tc_content = entry