Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
feat(granitemoe): Add support for "granitemoe" architecture
Gabe Goodhartfc90175c00bb
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 2dac8666..74eb6f1f 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -179,6 +179,7 @@ enum llm_arch {LLM_ARCH_NEMOTRON,LLM_ARCH_EXAONE,LLM_ARCH_GRANITE,+ LLM_ARCH_GRANITE_MOE,LLM_ARCH_UNKNOWN,};@@ -227,6 +228,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_NEMOTRON, "nemotron" },{ LLM_ARCH_EXAONE, "exaone" },{ LLM_ARCH_GRANITE, "granite" },+ { LLM_ARCH_GRANITE_MOE, "granitemoe" },{ LLM_ARCH_UNKNOWN, "(unknown)" },};@@ -1328,6 +1330,24 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_GRANITE_MOE,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },+ },+ },{LLM_ARCH_UNKNOWN,{@@ -1956,7 +1976,7 @@ struct llama_hparams {float f_max_alibi_bias = 0.0f;float f_logit_scale = 0.0f;- // Additional scale factors (Granite)+ // Additional scale factors (Granite/Granite MoE)float f_residual_scale = 0.0f;float f_embedding_scale = 0.0f;float f_attention_scale = 0.0f;@@ -5032,6 +5052,7 @@ static void llm_load_hparams(}} break;case LLM_ARCH_GRANITE:+ case LLM_ARCH_GRANITE_MOE:{ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);@@ -5040,6 +5061,7 @@ static void llm_load_hparams(ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);switch (hparams.n_layer) {+ case 32: model.type = e_model::MODEL_3B; break;case 40: model.type = e_model::MODEL_3B; break;// Add additional layer/vocab/etc checks here for other model sizesdefault: model.type = e_model::MODEL_UNKNOWN;@@ -5713,7 +5735,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);}- if (model.arch == LLM_ARCH_GRANITE) {+ if (model.arch == LLM_ARCH_GRANITE || model.arch == LLM_ARCH_GRANITE_MOE) {LLAMA_LOG_INFO("%s: f_embedding_scale = %f\n", __func__, hparams.f_embedding_scale);LLAMA_LOG_INFO("%s: f_residual_scale = %f\n", __func__, hparams.f_residual_scale);LLAMA_LOG_INFO("%s: f_attention_scale = %f\n", __func__, hparams.f_attention_scale);@@ -5891,6 +5913,7 @@ static bool llm_load_tensors(case LLM_ARCH_REFACT:case LLM_ARCH_MINICPM:case LLM_ARCH_GRANITE:+ case LLM_ARCH_GRANITE_MOE:{model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});@@ -14019,6 +14042,7 @@ static struct ggml_cgraph * llama_build_graph(switch (model.arch) {case LLM_ARCH_LLAMA:case LLM_ARCH_GRANITE:+ case LLM_ARCH_GRANITE_MOE:{result = llm.build_llama();} break;@@ -17273,6 +17297,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_DEEPSEEK2:case LLM_ARCH_CHATGLM:case LLM_ARCH_GRANITE:+ case LLM_ARCH_GRANITE_MOE:return LLAMA_ROPE_TYPE_NORM;// the pairs of head values are offset by n_rot/2