LLM inference in C/C++
C++ ggml
vocab : refactor normalizer flags into options struct, add strip_accents (#24371)
o7si68f30663cf49
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 5845940973..bb79271950 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -272,7 +272,8 @@ class Keys:CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"CHAT_TEMPLATES = "tokenizer.chat_templates"# Normalizer constants- NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"+ NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"+ NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"# FIM/Infill special tokens constantsFIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 182c9c54a5..f707f29dc5 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1124,6 +1124,9 @@ class GGUFWriter:def add_normalizer_lowercase(self, value: bool) -> None:self.add_bool(Keys.Tokenizer.NORMALIZER_LOWERCASE, value)+ def add_normalizer_strip_accents(self, value: bool) -> None:+ self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value)+def add_eot_token_id(self, id: int) -> None:self.add_uint32(Keys.Tokenizer.EOT_ID, id)diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.pyindex 27d3845852..f8d3b3e740 100644--- a/gguf-py/gguf/vocab.py+++ b/gguf-py/gguf/vocab.py@@ -53,6 +53,7 @@ class SpecialVocab:special_token_ids: dict[str, int]chat_template: str | Sequence[Mapping[str, str]] | Nonenormalizer_lowercase: bool | None+ normalizer_strip_accents: bool | Nonedef __init__(self, path: str | os.PathLike[str], load_merges: bool = False,@@ -66,6 +67,7 @@ class SpecialVocab:self.merges = []self.chat_template = Noneself.normalizer_lowercase = None+ self.normalizer_strip_accents = Noneif special_token_types is not None:self.special_token_types = special_token_typeselse:@@ -108,6 +110,10 @@ class SpecialVocab:if not quiet:logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')gw.add_normalizer_lowercase(self.normalizer_lowercase)+ if self.normalizer_strip_accents is not None:+ if not quiet:+ logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')+ gw.add_normalizer_strip_accents(self.normalizer_strip_accents)def _load(self, path: Path) -> None:self._try_load_from_tokenizer_json(path)@@ -155,17 +161,21 @@ class SpecialVocab:def _parse_normalizer(self, normalizer: dict) -> None:# ref: https://huggingface.co/docs/tokenizers/api/normalizers#- # Detects lowercase normalization in three possible formats:- # 1. Standalone: {"type": "Lowercase"}- # 2. BertNormalizer attribute: {"type": "BertNormalizer", "lowercase": true, ...}- # 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}+ # Extracts normalizer flags from three possible formats:+ # 1. Standalone: {"type": "Lowercase"}+ # 2. BertNormalizer attrs: {"type": "BertNormalizer", ...}+ # 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}normalizer_type = normalizer.get('type')if normalizer_type == 'Lowercase':self.normalizer_lowercase = True+ elif normalizer_type == 'StripAccents':+ self.normalizer_strip_accents = Trueelif normalizer_type == 'BertNormalizer':if 'lowercase' in normalizer:self.normalizer_lowercase = normalizer['lowercase']+ if 'strip_accents' in normalizer:+ self.normalizer_strip_accents = normalizer['strip_accents']elif normalizer_type == 'Sequence':for norm in normalizer.get('normalizers', []):self._parse_normalizer(norm)diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 680b5fc64d..6172363cd0 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -299,39 +299,40 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_DENSE_3_FEAT_IN, "%s.dense_3_feat_in" },{ LLM_KV_DENSE_3_FEAT_OUT, "%s.dense_3_feat_out" },- { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },- { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },- { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },- { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },- { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },- { LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },- { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },- { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },- { LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },- { LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },- { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },- { LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },- { LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },- { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },- { LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },- { LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },- { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },- { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },- { LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" },- { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },- { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },- { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },- { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },- { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },- { LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" },- { LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },- { LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },- { LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },- { LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },- { LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" },- { LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" },- { LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" },- { LLM_KV_TOKENIZER_SUPPRESS_TOKENS, "tokenizer.ggml.suppress_tokens" },+ { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },+ { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },+ { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },+ { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },+ { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },+ { LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },+ { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },+ { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },+ { LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },+ { LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },+ { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },+ { LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },+ { LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },+ { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },+ { LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },+ { LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },+ { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },+ { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },+ { LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" },+ { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },+ { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },+ { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },+ { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },+ { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },+ { LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" },+ { LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },+ { LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" },+ { LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },+ { LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },+ { LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },+ { LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" },+ { LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" },+ { LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" },+ { LLM_KV_TOKENIZER_SUPPRESS_TOKENS, "tokenizer.ggml.suppress_tokens" },{ LLM_KV_ADAPTER_TYPE, "adapter.type" },{ LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },diff --git a/src/llama-arch.h b/src/llama-arch.hindex b65fce72e6..f663a3edba 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -314,6 +314,7 @@ enum llm_kv {LLM_KV_TOKENIZER_RWKV,LLM_KV_TOKENIZER_CHAT_TEMPLATE,LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,+ LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,LLM_KV_TOKENIZER_FIM_PRE_ID,LLM_KV_TOKENIZER_FIM_SUF_ID,LLM_KV_TOKENIZER_FIM_MID_ID,diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex 9a4bed4948..8543e178db 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -764,7 +764,7 @@ struct llm_tokenizer_wpm_session {void tokenize(const std::string & text, std::vector<llama_token> & output) {// normalize and split by whitespace- std::vector<std::string> words = preprocess(text, vocab.get_normalizer_lowercase());+ std::vector<std::string> words = preprocess(text, vocab.get_normalizer_opts());// bos token prepended already// find the longest tokens that form the words@@ -809,11 +809,14 @@ struct llm_tokenizer_wpm_session {}// TODO: reduce string copies by using cpts_offs array- static std::vector<std::string> preprocess(const std::string & text, bool lowercase) {- const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));+ static std::vector<std::string> preprocess(const std::string & text, const llama_vocab::normalizer_options & normalizer_opts) {+ std::vector<uint32_t> cpts = unicode_cpts_from_utf8(text);+ if (normalizer_opts.strip_accents) {+ cpts = unicode_cpts_normalize_nfd(cpts);+ }std::vector<std::string> words(1, "");- for (const uint32_t cpt : cpts_nfd) {+ for (const uint32_t cpt : cpts) {const auto flags = unicode_cpt_flags_from_cpt(cpt);if (flags.is_whitespace) {@@ -828,7 +831,11 @@ struct llm_tokenizer_wpm_session {continue;}- const std::string s = unicode_cpt_to_utf8(lowercase ? unicode_tolower(cpt) : cpt);+ if (normalizer_opts.strip_accents && flags.is_accent_mark) {+ continue;+ }++ const std::string s = unicode_cpt_to_utf8(normalizer_opts.lowercase ? unicode_tolower(cpt) : cpt);if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) {if (words.back().size()) { // finish previous word if anywords.emplace_back();@@ -1692,7 +1699,7 @@ struct llm_tokenizer_whitespace_session : llm_tokenizer_bpe_session {llm_tokenizer_whitespace_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : llm_tokenizer_bpe_session{vocab, tokenizer}, vocab{vocab} {}void tokenize(const std::string & text, std::vector<llama_token> & output) override {- const bool lowercase = vocab.get_normalizer_lowercase();+ const bool lowercase = vocab.get_normalizer_opts().lowercase;std::string segment;auto flush = [&]() {@@ -1797,7 +1804,9 @@ struct llama_vocab::impl {bool remove_extra_whitespaces = false;bool escape_whitespaces = true;bool treat_whitespace_as_suffix = false;- bool normalizer_lowercase = true; // Lowercase normalizer (tokenizer.json)++ // BertNormalizer options+ llama_vocab::normalizer_options normalizer_opts;std::unordered_map<std::string, llama_token> token_to_id;std::vector<token_data> id_to_token;@@ -2172,7 +2181,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {} else if (tokenizer_pre == "whitespace") {pre_type = LLAMA_VOCAB_PRE_TYPE_WHITESPACE;- normalizer_lowercase = false;+ normalizer_opts.lowercase = false;} else if (tokenizer_pre == "refact") {pre_type = LLAMA_VOCAB_PRE_TYPE_REFACT;@@ -2532,8 +2541,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {}}- // Lowercase normalizer flag (consulted by WPM / whitespace BPE)- ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, normalizer_lowercase, false);+ // BertNormalizer options+ ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, normalizer_opts.lowercase, false);+ normalizer_opts.strip_accents = normalizer_opts.lowercase;+ ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false);// suppress tokens{@@ -3969,8 +3980,8 @@ bool llama_vocab::get_treat_whitespace_as_suffix() const {return pimpl->treat_whitespace_as_suffix;}-bool llama_vocab::get_normalizer_lowercase() const {- return pimpl->normalizer_lowercase;+const llama_vocab::normalizer_options & llama_vocab::get_normalizer_opts() const {+ return pimpl->normalizer_opts;}const std::vector<llama_token> & llama_vocab::get_suppress_tokens() const {diff --git a/src/llama-vocab.h b/src/llama-vocab.hindex 2626ae36e3..707cd4bac4 100644--- a/src/llama-vocab.h+++ b/src/llama-vocab.h@@ -76,6 +76,12 @@ struct llama_vocab {llama_token_attr attr;};+ struct normalizer_options {+ bool lowercase = true;+ bool strip_accents = true;+ // TODO: clean_text, handle_chinese_chars+ };+llama_vocab();~llama_vocab();@@ -141,7 +147,7 @@ struct llama_vocab {bool get_remove_extra_whitespaces () const;bool get_escape_whitespaces () const;bool get_treat_whitespace_as_suffix() const;- bool get_normalizer_lowercase () const;+ const normalizer_options & get_normalizer_opts() const;const std::vector<llama_token> & get_suppress_tokens() const;