Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add smaug-bpe tokenizer
Justine Tunney7469a23735cf
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 7e92fa98..bf1c62d4 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -4321,6 +4321,9 @@ static void llm_load_vocab(} else if (tokenizer_pre == "dbrx") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DBRX;+ } else if (+ tokenizer_pre == "smaug-bpe") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG;} else if (tokenizer_pre == "smollm") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMOLLM;@@ -12143,6 +12146,7 @@ struct llm_tokenizer_bpe {});break;case LLAMA_VOCAB_PRE_TYPE_DBRX:+ case LLAMA_VOCAB_PRE_TYPE_SMAUG:word_collection = unicode_regex_split(text, {// same as llama3"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 78b4890c..f258b0bd 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -87,7 +87,8 @@ extern "C" {LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,LLAMA_VOCAB_PRE_TYPE_OLMO = 12,LLAMA_VOCAB_PRE_TYPE_DBRX = 13,- LLAMA_VOCAB_PRE_TYPE_SMOLLM = 14,+ LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,+ LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,};// note: these values should be synchronized with ggml_rope