LLM inference in C/C++
C++ ggml
gguf-py : detect nfc normalization (#30540)
Sigbjørn Skjæret683bbc0dce02
diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.pyindex 31036bbe5f..78e90eddc4 100644--- a/gguf-py/gguf/vocab.py+++ b/gguf-py/gguf/vocab.py@@ -52,6 +52,7 @@ class SpecialVocab:add_special_token: dict[str, bool]special_token_ids: dict[str, int]chat_template: str | Sequence[Mapping[str, str]] | None+ normalizer_nfc: bool | Nonenormalizer_lowercase: bool | Nonenormalizer_strip_accents: bool | None@@ -66,6 +67,7 @@ class SpecialVocab:self.load_merges = load_mergesself.merges = []self.chat_template = None+ self.normalizer_nfc = Noneself.normalizer_lowercase = Noneself.normalizer_strip_accents = Noneif special_token_types is not None:@@ -106,6 +108,10 @@ class SpecialVocab:if not quiet:logger.info(f'Setting chat_template to {self.chat_template}')gw.add_chat_template(self.chat_template)+ if self.normalizer_nfc is not None:+ if not quiet:+ logger.info(f'Setting normalizer_nfc to {self.normalizer_nfc}')+ gw.add_normalizer_nfc(self.normalizer_nfc)if self.normalizer_lowercase is not None:if not quiet:logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')@@ -169,6 +175,8 @@ class SpecialVocab:normalizer_type = normalizer.get('type')if normalizer_type == 'Lowercase':self.normalizer_lowercase = True+ elif normalizer_type == 'NFC':+ self.normalizer_nfc = Trueelif normalizer_type == 'StripAccents':self.normalizer_strip_accents = Trueelif normalizer_type == 'BertNormalizer':