LLM inference in C/C++
C++ ggml
nemotron : support non-standard GQA, BPE tokenizer and domyn-small pre-type (#23343)
martin-cimmino0693dbfb915f
diff --git a/conversion/base.py b/conversion/base.pyindex f3bbe7b0ff..d8420eb325 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -1837,6 +1837,9 @@ class TextModel(ModelBase):if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":# ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0res = "minerva-7b"+ if chkhsh == "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262":+ # ref: https://huggingface.co/domyn/Domyn-Small-v1.0+ res = "domyn-small"if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664":# ref: https://huggingface.co/tencent/Hunyuan-A13B-Instructres = "hunyuan"diff --git a/conversion/nemotron.py b/conversion/nemotron.pyindex c900481943..d76a45d828 100644--- a/conversion/nemotron.py+++ b/conversion/nemotron.py@@ -1,5 +1,6 @@from __future__ import annotations+import refrom typing import Any, Callable, Iterable, Sequence, TYPE_CHECKINGimport numpy as np@@ -367,9 +368,21 @@ class NemotronModel(TextModel):model_arch = gguf.MODEL_ARCH.NEMOTRONdef set_vocab(self):- self._set_vocab_sentencepiece()- self.gguf_writer.add_pad_token_id(0)- self.gguf_writer.add_unk_token_id(1)+ if (self.dir_model / "tokenizer.model").exists():+ self._set_vocab_sentencepiece()+ self.gguf_writer.add_pad_token_id(0)+ self.gguf_writer.add_unk_token_id(1)+ else:+ # SPM-style BPE (Metaspace + byte fallback): mark <0xXX> tokens as BYTE so they decode to raw bytes+ tokens, toktypes, tokpre = self.get_vocab_base()+ toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)]+ self.gguf_writer.add_tokenizer_model("gpt2")+ self.gguf_writer.add_tokenizer_pre(tokpre)+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_types(toktypes)++ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)+ special_vocab.add_to_gguf(self.gguf_writer)def set_gguf_parameters(self):super().set_gguf_parameters()@@ -383,7 +396,10 @@ class NemotronModel(TextModel):rot_pct = self.rope_parameters["partial_rotary_factor"]n_embd = self.find_hparam(["hidden_size", "n_embd"])n_head = self.find_hparam(["num_attention_heads", "n_head"])- self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)+ # Use explicit head_dim when available; fallback keeps standard Nemotron working.+ # Without this, models where n_head * head_dim != n_embd (e.g. 48 heads, head_dim=128) produce the wrong rope dimension count.+ head_dim = self.hparams.get("head_dim") or n_embd // n_head+ self.gguf_writer.add_rope_dimension_count(int(rot_pct * head_dim))# * RopeScaling for Nemotronfactor = self.hparams.get("factor") or self.rope_parameters.get("factor")diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.pyindex 7d8c4fda64..37df033ef3 100755--- a/convert_hf_to_gguf_update.py+++ b/convert_hf_to_gguf_update.py@@ -178,6 +178,7 @@ pre_computed_hashes = [{"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.5-Air", "chkhsh": "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902"},{"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"},{"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"},+ {"name": "domyn-small", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/domyn/Domyn-Small-v1.0", "chkhsh": "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262"},{"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"},{"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"},{"name": "hy_v4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hy4-preview", "chkhsh": "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c"},diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 4482f7f335..0b95580bd2 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -383,6 +383,8 @@ class Keys:NORMALIZER_NFC = "tokenizer.ggml.normalizer.nfc"NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"+ # Metaspace constants+ METASPACE_PREPEND_SCHEME = "tokenizer.ggml.metaspace.prepend_scheme"# FIM/Infill special tokens constantsFIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 2faaefa59d..9e49a15140 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1377,6 +1377,9 @@ class GGUFWriter:def add_normalizer_strip_accents(self, value: bool) -> None:self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value)+ def add_metaspace_prepend_scheme(self, value: str) -> None:+ self.add_string(Keys.Tokenizer.METASPACE_PREPEND_SCHEME, value)+def add_eot_token_id(self, id: int) -> None:self.add_uint32(Keys.Tokenizer.EOT_ID, id)diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.pyindex 6383178ea7..3b82c39614 100644--- a/gguf-py/gguf/vocab.py+++ b/gguf-py/gguf/vocab.py@@ -56,6 +56,7 @@ class SpecialVocab:normalizer_lowercase: bool | Nonenormalizer_strip_accents: bool | Noneclean_spaces: bool | None+ metaspace_prepend_scheme: str | Nonedef __init__(self, path: str | os.PathLike[str], load_merges: bool = False,@@ -72,6 +73,7 @@ class SpecialVocab:self.normalizer_lowercase = Noneself.normalizer_strip_accents = Noneself.clean_spaces = None+ self.metaspace_prepend_scheme = Noneif special_token_types is not None:self.special_token_types = special_token_typeselse:@@ -126,6 +128,10 @@ class SpecialVocab:if not quiet:logger.info(f'Setting clean_spaces to {self.clean_spaces}')gw.add_clean_spaces(self.clean_spaces)+ if self.metaspace_prepend_scheme is not None:+ if not quiet:+ logger.info(f'Setting metaspace_prepend_scheme to {self.metaspace_prepend_scheme}')+ gw.add_metaspace_prepend_scheme(self.metaspace_prepend_scheme)def _load(self, path: Path) -> None:self._try_load_from_tokenizer_json(path)@@ -194,6 +200,16 @@ class SpecialVocab:for norm in normalizer.get('normalizers', []):self._parse_normalizer(norm)+ def _parse_pre_tokenizer(self, pre_tokenizer: dict) -> None:+ # ref: https://huggingface.co/docs/tokenizers/api/pre-tokenizers+ pre_tokenizer_type = pre_tokenizer.get('type')+ if pre_tokenizer_type == 'Metaspace':+ if 'prepend_scheme' in pre_tokenizer:+ self.metaspace_prepend_scheme = pre_tokenizer['prepend_scheme']+ elif pre_tokenizer_type == 'Sequence':+ for pre in pre_tokenizer.get('pretokenizers', []):+ self._parse_pre_tokenizer(pre)+def _try_load_from_tokenizer_json(self, path: Path) -> bool:tokenizer = Nonetokenizer_file = path / 'tokenizer.json'@@ -229,6 +245,8 @@ class SpecialVocab:# Parse normalizer configuration (e.g. Lowercase) into metadataif normalizer := tokenizer.get('normalizer'):self._parse_normalizer(normalizer)+ if pre_tokenizer := tokenizer.get('pre_tokenizer'):+ self._parse_pre_tokenizer(pre_tokenizer)added_tokens = tokenizer.get('added_tokens', {})else:added_tokens = {}diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex d79b18dec9..a364174c1b 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -445,6 +445,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_TOKENIZER_NORMALIZER_NFC, "tokenizer.ggml.normalizer.nfc" },{ LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },{ LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" },+ { LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, "tokenizer.ggml.metaspace.prepend_scheme" },{ LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },{ LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },{ LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },diff --git a/src/llama-arch.h b/src/llama-arch.hindex 12d8fe3925..cdf978fb83 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -408,6 +408,7 @@ enum llm_kv {LLM_KV_TOKENIZER_NORMALIZER_NFC,LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,+ LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME,LLM_KV_TOKENIZER_FIM_PRE_ID,LLM_KV_TOKENIZER_FIM_SUF_ID,LLM_KV_TOKENIZER_FIM_MID_ID,diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex 80a5fcdff8..78512bef57 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -543,6 +543,7 @@ struct llm_tokenizer_bpe : llm_tokenizer {byte_encode = false; // uses raw UTF-8, not GPT-2 byte encodingbreak;case LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE:+ case LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL:// Sarvam uses SPM-style BPE (same shape as Gemma4): spaces replaced with U+2581// by the normalizer, BPE merges over the whole text on raw UTF-8.regex_exprs = {@@ -2097,6 +2098,8 @@ struct llama_vocab::impl {// Tokenizer normalizer optionsllama_vocab::normalizer_options normalizer_opts;+ llama_metaspace_prepend_scheme metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER;+std::unordered_map<std::string, llama_token> token_to_id;std::vector<token_data> id_to_token;@@ -2612,6 +2615,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {} else if (tokenizer_pre == "minerva-7b") {pre_type = LLAMA_VOCAB_PRE_TYPE_MINERVA;+ } else if (+ tokenizer_pre == "domyn-small") {+ pre_type = LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL;+ escape_whitespaces = true;} else if (tokenizer_pre == "megrez") {pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;@@ -2956,6 +2963,24 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {normalizer_opts.strip_accents = normalizer_opts.lowercase;ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false);+ {+ std::string scheme;+ if (ml.get_key(LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, scheme, false)) {+ if (scheme == "never") {+ metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER;+ } else if (scheme == "first") {+ metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_FIRST;+ } else if (scheme == "always") {+ metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS;+ } else {+ throw std::runtime_error(format("unknown metaspace prepend_scheme: '%s'", scheme.c_str()));+ }+ if (type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS) {+ throw std::runtime_error("metaspace prepend_scheme 'always' is not supported for BPE vocab");+ }+ }+ }+// suppress tokens{const int suppress_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SUPPRESS_TOKENS).c_str());@@ -3743,6 +3768,7 @@ std::vector<llama_token> llama_vocab::impl::tokenize(// tokenizer.encode('', add_special_tokens=True) returns [1]// tokenizer.encode('', add_special_tokens=False) returns []+ // TODO: honor metaspace_prepend_scheme (add_space_prefix currently behaves like "always")bool is_prev_special = true; // prefix with space if first tokenif (add_special && add_bos) {@@ -3813,6 +3839,13 @@ std::vector<llama_token> llama_vocab::impl::tokenize(llama_escape_whitespace(text);}+ // Metaspace prepend_scheme "first": add U+2581 only at the start of the input+ static const std::string escaped_space = "\xE2\x96\x81";+ if (metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST &&+ &fragment == &fragment_buffer.front() && text.rfind(escaped_space, 0) != 0) {+ text.insert(0, escaped_space);+ }+#ifdef PRETOKENIZERDEBUGLLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", text.length(), fragment.offset, fragment.length, text.c_str());#endif@@ -3860,6 +3893,7 @@ std::vector<llama_token> llama_vocab::impl::tokenize(GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL);output.push_back(special_bos_id);}+ // TODO: honor metaspace_prepend_schemellm_tokenizer_ugm_session session(vocab, *static_cast<const llm_tokenizer_ugm *>(tokenizer.get()));for (const auto & fragment : fragment_buffer) {@@ -4143,10 +4177,17 @@ int32_t llama_vocab::impl::detokenize(}}+ // Metaspace prepend_scheme "first": remove the leading space of the first rendered piece, so a skipped BOS does not consume it+ // TODO: honor metaspace_prepend_scheme for the other vocab types+ bool remove_first_space = type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST;+for (int32_t i = 0; i < n_tokens; ++i) {GGML_ASSERT(avail >= 0);- int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space, unparse_special);+ int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space || remove_first_space, unparse_special);remove_space = false;+ if (unparse_special || !(token_get_attr(tokens[i]) & (LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL))) {+ remove_first_space = false;+ }if (n_chars < 0) {avail = 0;total -= n_chars;diff --git a/src/llama-vocab.h b/src/llama-vocab.hindex a87c0de42b..40357dba7e 100644--- a/src/llama-vocab.h+++ b/src/llama-vocab.h@@ -71,6 +71,14 @@ enum llama_vocab_pre_type {LLAMA_VOCAB_PRE_TYPE_MMBERT = 60,LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61,LLAMA_VOCAB_PRE_TYPE_FLINT = 62,+ LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL = 63,+};++// HF Metaspace pre-tokenizer prepend_scheme+enum llama_metaspace_prepend_scheme {+ LLAMA_METASPACE_PREPEND_SCHEME_NEVER = 0,+ LLAMA_METASPACE_PREPEND_SCHEME_FIRST = 1,+ LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS = 2,};struct LLM_KV;diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cppindex 129cf622e2..6b5b771074 100644--- a/src/models/nemotron.cpp+++ b/src/models/nemotron.cpp@@ -25,8 +25,11 @@ void llama_model_nemotron::load_arch_tensors(llama_model_loader &) {layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);- create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);- layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);+ // n_embd_head_k * n_head equals n_embd for standard Nemotron (32 heads),+ // but differs for models with non-standard GQA (e.g. 48 Q heads, head_dim=128).+ const int64_t n_embd_q = n_embd_head_k * n_head;+ create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_gqa, n_embd_gqa, 0);+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);// optional bias tensorslayer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);