LLM inference in C/C++
C++ ggml
vocab : add ufakzeka pre-tokenizer (#29033)
Sait Furkan Tekedc85f89c7ee2
diff --git a/conversion/base.py b/conversion/base.pyindex 8f6b3519cb..6aca7f1d34 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -1861,6 +1861,9 @@ class TextModel(ModelBase):if chkhsh == "972da7b59cec44d1f0a490a86c96df53859e486e481563e5dddac155013d87ac":# ref: https://huggingface.co/poolside/Laguna-XS.2res = "laguna"+ if chkhsh == "653660222fb704f61cbf2b618a8ae6502b7f8b20c980f9a5de07ed78e13319cd":+ # ref: https://huggingface.co/ufakai/ufakzeka-1+ res = "ufakzeka"if res is None:logger.warning("\n")diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.pyindex 3a15a6fca3..24b9bc0757 100755--- a/convert_hf_to_gguf_update.py+++ b/convert_hf_to_gguf_update.py@@ -163,6 +163,7 @@ models = [{"name": "granite-embed-multi-311m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-311m-multilingual-r2", },{"name": "mellum2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base"},{"name": "laguna", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/poolside/Laguna-XS.2", },+ {"name": "ufakzeka", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ufakai/ufakzeka-1", },]# some models are known to be broken upstream, so we will skip them as exceptionsdiff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex ee65faf23e..737e072756 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -488,6 +488,12 @@ struct llm_tokenizer_bpe : llm_tokenizer {"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1}| ?[^\\s\\p{L}\\p{N}\\r\\n]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",};break;+ case LLAMA_VOCAB_PRE_TYPE_UFAKZEKA:+ regex_exprs = {+ // Qwen2 pattern without the English contraction group, so Turkish apostrophe suffixes stay attached+ "[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",+ };+ break;case LLAMA_VOCAB_PRE_TYPE_GROK_2:regex_exprs = {// original regex from tokenizer.json@@ -2376,6 +2382,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {tokenizer_pre == "kimi-k2") {pre_type = LLAMA_VOCAB_PRE_TYPE_KIMI_K2;clean_spaces = false;+ } else if (+ tokenizer_pre == "ufakzeka") {+ pre_type = LLAMA_VOCAB_PRE_TYPE_UFAKZEKA;+ clean_spaces = false;} else if (tokenizer_pre == "grok-2") {pre_type = LLAMA_VOCAB_PRE_TYPE_GROK_2;diff --git a/src/llama-vocab.h b/src/llama-vocab.hindex 65293c0261..3fb061f0ec 100644--- a/src/llama-vocab.h+++ b/src/llama-vocab.h@@ -67,6 +67,7 @@ enum llama_vocab_pre_type {LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56,LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57,LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58,+ LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59,};struct LLM_KV;