Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
add phi4 support
CJ Paisa4ece762cae2
diff --git a/llama.cpp/llama-vocab.cpp b/llama.cpp/llama-vocab.cppindex bf0d24d4..bb9d82e7 100644--- a/llama.cpp/llama-vocab.cpp+++ b/llama.cpp/llama-vocab.cpp@@ -438,6 +438,13 @@ struct llm_tokenizer_bpe {"[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",};break;+ case LLAMA_VOCAB_PRE_TYPE_GPT4O:+ // original regex from tokenizer.json+ // "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",+ regex_exprs = {+ "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",+ };+ break;default:// default regex for BPE tokenization pre-processingregex_exprs = {diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 1499a827..6da5fc19 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -5364,6 +5364,10 @@ static void llm_load_vocab(} else if (tokenizer_pre == "exaone") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_EXAONE;+ } else if (+ tokenizer_pre == "gpt-4o") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT4O;+ vocab.tokenizer_clean_spaces = false;} else {throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));}@@ -5946,6 +5950,7 @@ static bool llm_load_tensors(const int64_t n_embd_gqa = n_embd_v_gqa;const int64_t n_vocab = hparams.n_vocab;const int64_t n_vocab_type = hparams.n_vocab_type;+ const int64_t n_rot = hparams.n_rot;const int64_t n_expert = hparams.n_expert;const int64_t n_expert_used = hparams.n_expert_used;const int64_t n_ctx_train = hparams.n_ctx_train;@@ -6661,7 +6666,12 @@ static bool llm_load_tensors(// output{model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd });- model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab });+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, llama_model_loader::TENSOR_NOT_REQUIRED);++ // if output is NULL, init from the input tok embed+ if (model.output == NULL) {+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, llama_model_loader::TENSOR_DUPLICATED);+ }}for (int i = 0; i < n_layer; ++i) {@@ -6680,8 +6690,8 @@ static bool llm_load_tensors(layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd });layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, 2 * n_ff });- layer.rope_long = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight"), { n_embd_head/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));- layer.rope_short = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight"), { n_embd_head/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));+ layer.rope_long = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight"), { n_rot/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));+ layer.rope_short = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight"), { n_rot/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));}} break;case LLM_ARCH_PLAMO:@@ -10869,7 +10879,13 @@ struct llm_build_context {struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa();+ struct ggml_tensor * KQ_mask = nullptr;+ if (hparams.n_swa == 0) {+ // Phi-4 doesn't use sliding window attention+ KQ_mask = build_inp_KQ_mask();+ } else {+ KQ_mask = build_inp_KQ_mask_swa();+ }for (int il = 0; il < n_layer; ++il) {auto residual = inpL;@@ -10927,7 +10943,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, lctx, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask_swa, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -19325,6 +19341,14 @@ static int32_t llama_chat_apply_template_internal(if (add_ass) {ss << "<|assistant|>\n";}+ } else if (tmpl == "phi4") {+ // chatml template+ for (auto message : chat) {+ ss << "<|im_start|>" << message->role << "<|im_sep|>" << message->content << "<|im_end|>";+ }+ if (add_ass) {+ ss << "<|im_start|>assistant<|im_sep|>";+ }} else if (tmpl == "zephyr" || tmpl_contains("<|user|>")) {// zephyr templatefor (auto message : chat) {diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex ea602059..35084576 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -98,6 +98,7 @@ extern "C" {LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,+ LLAMA_VOCAB_PRE_TYPE_GPT4O = 26,};enum llama_rope_type {