Skip to content

LLM inference in C/C++

C++ ggml

model : load MiMo V2 MTP tensors only if used (#26412)

Sigbjørn Skjæret75587a05b39e
diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp
index 4080a934cb..d50e186cce 100644
--- a/src/models/mimo2.cpp
+++ b/src/models/mimo2.cpp
@@ -30,7 +30,11 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
- const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
+ int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
+
+ if (!ml.load_mtp) {
+ mtp_flags |= TENSOR_SKIP;
+ }
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);