LLM inference in C/C++
C++ ggml
vocab : fix Nemotron-3.5-Embed tokenization (#30974)
Takamichi Oharaf9599d0cc899
diff --git a/conversion/mistral3.py b/conversion/mistral3.pyindex 106c329e72..6c771b26f3 100644--- a/conversion/mistral3.py+++ b/conversion/mistral3.py@@ -39,6 +39,11 @@ class Mistral3Model(TextModel):self.gguf_writer.add_add_bos_token(False)self.gguf_writer.add_add_eos_token(False)+ def get_vocab_base_pre(self, tokenizer) -> str:+ if self.origin_hf_arch == "Mistral3Model":+ return "nemotron-3.5-embed"+ return super().get_vocab_base_pre(tokenizer)+def set_gguf_parameters(self):super().set_gguf_parameters()self._try_set_pooling_type()diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex 78512bef57..f72fbd2215 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -422,6 +422,7 @@ struct llm_tokenizer_bpe : llm_tokenizer {};break;case LLAMA_VOCAB_PRE_TYPE_TEKKEN:+ case LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD:// original regex from tokenizer.json// "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"regex_exprs = {@@ -637,11 +638,22 @@ struct llm_tokenizer_bpe_session {}virtual void tokenize(const std::string & text, std::vector<llama_token> & output) {+ auto tok_pre = vocab.get_pre_type();++ std::vector<std::string> word_collection;+ if (tok_pre == LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD) {+ static const std::string zwnj = "\xe2\x80\x8c";+ std::string normalized_text = text;+ replace_all(normalized_text, "[" + zwnj, "[");+ replace_all(normalized_text, "<" + zwnj, "<");+ word_collection = unicode_regex_split(normalized_text, tokenizer.regex_exprs, tokenizer.byte_encode);+ } else {+ word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode);+ }+int final_prev_index = -1;- auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode);symbols_final.clear();- auto tok_pre = vocab.get_pre_type();if (tok_pre == LLAMA_VOCAB_PRE_TYPE_MMBERT) {// Metaspace pre-tokenizer: a text starts with an escaped space, and each escaped space starts a new word@@ -2713,6 +2725,11 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {tokenizer_pre == "k2-horizon") {pre_type = LLAMA_VOCAB_PRE_TYPE_K2_HORIZON;clean_spaces = false;+ } else if (+ tokenizer_pre == "nemotron-3.5-embed") {+ pre_type = LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD;+ ignore_merges = true;+ add_bos = true;} else {throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));}diff --git a/src/llama-vocab.h b/src/llama-vocab.hindex 40357dba7e..c6f3ac06dc 100644--- a/src/llama-vocab.h+++ b/src/llama-vocab.h@@ -72,6 +72,7 @@ enum llama_vocab_pre_type {LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61,LLAMA_VOCAB_PRE_TYPE_FLINT = 62,LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL = 63,+ LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64,};// HF Metaspace pre-tokenizer prepend_scheme