LLM inference in C/C++
C++ ggml
vocab : add clean_spaces GGUF key (#30940)
tkran2d88040c704c8
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 51a16ef52b..4482f7f335 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -371,6 +371,7 @@ class Keys:DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags"ADD_PREFIX = "tokenizer.ggml.add_space_prefix"REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces"+ CLEAN_SPACES = "tokenizer.ggml.clean_spaces"PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens"HF_JSON = "tokenizer.huggingface.json"diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex d003922116..2faaefa59d 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1326,6 +1326,9 @@ class GGUFWriter:def add_remove_extra_whitespaces(self, value: bool) -> None:self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value)+ def add_clean_spaces(self, value: bool) -> None:+ self.add_bool(Keys.Tokenizer.CLEAN_SPACES, value)+def add_precompiled_charsmap(self, charsmap: bytes) -> None:self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap)diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.pyindex 78e90eddc4..6383178ea7 100644--- a/gguf-py/gguf/vocab.py+++ b/gguf-py/gguf/vocab.py@@ -55,6 +55,7 @@ class SpecialVocab:normalizer_nfc: bool | Nonenormalizer_lowercase: bool | Nonenormalizer_strip_accents: bool | None+ clean_spaces: bool | Nonedef __init__(self, path: str | os.PathLike[str], load_merges: bool = False,@@ -70,6 +71,7 @@ class SpecialVocab:self.normalizer_nfc = Noneself.normalizer_lowercase = Noneself.normalizer_strip_accents = None+ self.clean_spaces = Noneif special_token_types is not None:self.special_token_types = special_token_typeselse:@@ -120,6 +122,10 @@ class SpecialVocab:if not quiet:logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')gw.add_normalizer_strip_accents(self.normalizer_strip_accents)+ if self.clean_spaces is not None:+ if not quiet:+ logger.info(f'Setting clean_spaces to {self.clean_spaces}')+ gw.add_clean_spaces(self.clean_spaces)def _load(self, path: Path) -> None:self._try_load_from_tokenizer_json(path)@@ -345,6 +351,10 @@ class SpecialVocab:self.chat_template = chat_templateelse:logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring')+ # decode clean-up of spaces before punctuation (transformers clean_up_tokenization_spaces)+ clean_spaces = tokenizer_config.get('clean_up_tokenization_spaces')+ if isinstance(clean_spaces, bool):+ self.clean_spaces = clean_spacesfor typ in self.special_token_types:add_entry = tokenizer_config.get(f'add_{typ}_token')if isinstance(add_entry, bool):diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 62b012c5c3..d79b18dec9 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -437,6 +437,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" },{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },+ { LLM_KV_TOKENIZER_CLEAN_SPACES, "tokenizer.ggml.clean_spaces" },{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },diff --git a/src/llama-arch.h b/src/llama-arch.hindex 6713db6b59..12d8fe3925 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -400,6 +400,7 @@ enum llm_kv {LLM_KV_TOKENIZER_DNA_AUTO_TAGS,LLM_KV_TOKENIZER_ADD_PREFIX,LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,+ LLM_KV_TOKENIZER_CLEAN_SPACES,LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,LLM_KV_TOKENIZER_HF_JSON,LLM_KV_TOKENIZER_RWKV,diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cppindex d620fcd9c8..2135b82b15 100644--- a/src/llama-model-saver.cpp+++ b/src/llama-model-saver.cpp@@ -424,6 +424,7 @@ void llama_model_saver::add_kv_from_model() {add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep());add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix());add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces());+ add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES, vocab.get_clean_spaces());add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());// add_kv(LLM_KV_TOKENIZER_HF_JSON, ???);// add_kv(LLM_KV_TOKENIZER_RWKV, ???);diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex d25c4ceec5..80a5fcdff8 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -2432,6 +2432,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {} else if (tokenizer_pre == "minicpm5") {pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5;ignore_merges = true;+ clean_spaces = false;} else if (tokenizer_pre == "llama3" ||tokenizer_pre == "llama-v3" ||@@ -2439,11 +2440,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {tokenizer_pre == "falcon3" ||tokenizer_pre == "falcon-h1" ||tokenizer_pre == "midm-2.0" ||- tokenizer_pre == "lfm2" ||tokenizer_pre == "jina-v5-nano") {pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;ignore_merges = true;add_bos = true;+ } else if (+ tokenizer_pre == "lfm2") {+ pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;+ ignore_merges = true;+ add_bos = true;+ clean_spaces = false;} else if (tokenizer_pre == "deepseek-llm") {pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;@@ -2494,6 +2500,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {tokenizer_pre == "granite-embed-multi-311m") {pre_type = LLAMA_VOCAB_PRE_TYPE_GEMMA4;escape_whitespaces = true;+ clean_spaces = false;} else if (tokenizer_pre == "mmbert") {pre_type = LLAMA_VOCAB_PRE_TYPE_MMBERT;@@ -2540,6 +2547,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {} else if (tokenizer_pre == "dbrx") {pre_type = LLAMA_VOCAB_PRE_TYPE_DBRX;+ clean_spaces = false;} else if (tokenizer_pre == "smaug-bpe") {pre_type = LLAMA_VOCAB_PRE_TYPE_SMAUG;@@ -2551,12 +2559,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {tokenizer_pre == "chatglm-bpe") {pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;special_bos_id = LLAMA_TOKEN_NULL;+ clean_spaces = false;} else if (tokenizer_pre == "glm4" ||tokenizer_pre == "glm5") {pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;special_bos_id = LLAMA_TOKEN_NULL;ignore_merges = true;+ clean_spaces = false;} else if (tokenizer_pre == "viking") {pre_type = LLAMA_VOCAB_PRE_TYPE_VIKING;@@ -2590,6 +2600,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {} else if (tokenizer_pre == "exaone4") {pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;+ clean_spaces = false;} else if (tokenizer_pre == "exaone-moe") {pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE;@@ -2690,6 +2701,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {} else if (tokenizer_pre == "mellum2") {pre_type = LLAMA_VOCAB_PRE_TYPE_MELLUM2;+ clean_spaces = false;} else if (tokenizer_pre == "k2-horizon") {pre_type = LLAMA_VOCAB_PRE_TYPE_K2_HORIZON;@@ -2726,6 +2738,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX, add_space_prefix, false);ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, remove_extra_whitespaces, false);+ ml.get_key(LLM_KV_TOKENIZER_CLEAN_SPACES, clean_spaces, false);}const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txtindex 24e6ba609a..815cc15a32 100644--- a/tests/CMakeLists.txt+++ b/tests/CMakeLists.txt@@ -142,7 +142,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r ARGS ${PROJEllama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf)llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf)llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf)-llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf)+llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf --check-decode)llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf)llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf)diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cppindex 37f8312c46..5253a46a61 100644--- a/tests/test-tokenizer-0.cpp+++ b/tests/test-tokenizer-0.cpp@@ -125,7 +125,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &int main(int argc, char **argv) {if (argc < 2) {- fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);+ fprintf(stderr, "Usage: %s vocab-file [text-file] [--check-decode]\n", argv[0]);return 1;}@@ -135,8 +135,13 @@ int main(int argc, char **argv) {const std::string fname_out = fname + ".out";std::string fname_text;- if (argc > 2) {- fname_text = argv[2];+ bool check_decode = false; // also check that the expected tokens detokenize to the input text+ for (int i = 2; i < argc; i++) {+ if (std::string(argv[i]) == "--check-decode") {+ check_decode = true;+ } else {+ fname_text = argv[i];+ }}fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());@@ -244,6 +249,14 @@ int main(int argc, char **argv) {success = false;}++ if (check_decode) {+ const std::string dec = common_detokenize(ctx, test_kv.second);+ if (dec != test_kv.first) {+ fprintf(stderr, "%s : failed decode: '%s' detokenized to '%s'\n", __func__, test_kv.first.c_str(), dec.c_str());+ success = false;+ }+ }}});}