Skip to content

Distribute and run LLMs with a single file.

Add smaug-bpe tokenizer

Justine Tunney7469a23735cf
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 7e92fa98..bf1c62d4 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -4321,6 +4321,9 @@ static void llm_load_vocab(
} else if (
tokenizer_pre == "dbrx") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DBRX;
+ } else if (
+ tokenizer_pre == "smaug-bpe") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG;
} else if (
tokenizer_pre == "smollm") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMOLLM;
@@ -12143,6 +12146,7 @@ struct llm_tokenizer_bpe {
});
break;
case LLAMA_VOCAB_PRE_TYPE_DBRX:
+ case LLAMA_VOCAB_PRE_TYPE_SMAUG:
word_collection = unicode_regex_split(text, {
// same as llama3
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index 78b4890c..f258b0bd 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -87,7 +87,8 @@ extern "C" {
LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,
LLAMA_VOCAB_PRE_TYPE_OLMO = 12,
LLAMA_VOCAB_PRE_TYPE_DBRX = 13,
- LLAMA_VOCAB_PRE_TYPE_SMOLLM = 14,
+ LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,
+ LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,
};
// note: these values should be synchronized with ggml_rope