LLM inference in C/C++
C++ ggml
model : add support for MiniMaxText01ForCausalLM and MiniMaxM1ForCausalLM (#27018)
fairydreaming16d222fc5ead
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 695289b73a..f4d475de7f 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -161,6 +161,8 @@ TEXT_MODEL_MAP: dict[str, str] = {"MiniCPM3ForCausalLM": "minicpm","MiniCPMForCausalLM": "minicpm","MiniCPMV4_6ForConditionalGeneration": "minicpm",+ "MiniMaxText01ForCausalLM": "minimax",+ "MiniMaxM1ForCausalLM": "minimax","MiniMaxM2ForCausalLM": "minimax","MiniMaxM3SparseForCausalLM": "minimax","MiniMaxM3SparseForConditionalGeneration": "minimax",diff --git a/conversion/minimax.py b/conversion/minimax.pyindex c2175cc932..d7a00bac95 100644--- a/conversion/minimax.py+++ b/conversion/minimax.py@@ -1,13 +1,121 @@from __future__ import annotations-from typing import TYPE_CHECKING+from typing import Iterable, Sequence, TYPE_CHECKINGimport torchif TYPE_CHECKING:from torch import Tensor-from .base import ModelBase, TextModel, MmprojModel, gguf+from .base import ModelBase, TextModel, MmprojModel, gguf, logger++[email protected]("MiniMaxText01ForCausalLM")[email protected]("MiniMaxM1ForCausalLM")+class MiniMaxText01Model(TextModel):+ model_arch = gguf.MODEL_ARCH.MINIMAX01++ def _get_suppress_tokens(self) -> Sequence[int] | None:+ import json+ from transformers import AutoTokenizer+ from .base import LazyTorchTensor++ # check added tokens embeddings in embeddings tensor for zero-valued embeddings+ # they get in the way of the token sampling process and must be suppressed++ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)+ tokenizer_vocab_size = tokenizer.vocab_size++ with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:+ weight_map = json.load(f)["weight_map"]++ embeddings_tensor_name = "model.embed_tokens.weight"+ embeddings_shard_name = weight_map[embeddings_tensor_name]+ with gguf.utility.SafetensorsLocal(self.dir_model / embeddings_shard_name) as model_shard:+ embeddings_data = model_shard[embeddings_tensor_name]++ embeddings_weights_dtype = LazyTorchTensor._dtype_str_map[embeddings_data.dtype]+ embeddings_weights = torch.from_numpy(embeddings_data.mmap_bytes()).view(embeddings_weights_dtype).reshape(embeddings_data.shape)+ embeddings_vocab_size = embeddings_weights.shape[0]++ embeddings_added_tokens = embeddings_weights[tokenizer_vocab_size:embeddings_vocab_size]+ embeddings_zero_rows = torch.all(embeddings_added_tokens == 0, dim=1)+ tokens_zero_embeddings_ids = (torch.nonzero(embeddings_zero_rows, as_tuple=False).flatten() + tokenizer_vocab_size).tolist()++ return tokens_zero_embeddings_ids++ def set_vocab(self) -> None:+ from pathlib import Path++ self._set_vocab_gpt2()++ for tmpl_file in [+ self.dir_model / "chat_template.jinja",+ Path(__file__).parent.parent / "models" / "templates" / "MiniMax-M1.jinja"+ ]:+ if tmpl_file.is_file():+ self.gguf_writer.add_chat_template(tmpl_file.read_text(encoding="utf-8"))+ logger.info(f"Chat template overridden with {tmpl_file}.")+ break++ def set_gguf_parameters(self):+ super().set_gguf_parameters()++ suppress_tokens = self._get_suppress_tokens()+ if suppress_tokens:+ logger.info(f"Suppressing tokens with zero embeddings {suppress_tokens}")+ self.gguf_writer.add_suppress_tokens(suppress_tokens)++ layernorm_full_attention_alpha = self.hparams["layernorm_full_attention_alpha"]+ layernorm_full_attention_beta = self.hparams["layernorm_full_attention_beta"]+ layernorm_linear_attention_alpha = self.hparams["layernorm_linear_attention_alpha"]+ layernorm_linear_attention_beta = self.hparams["layernorm_linear_attention_beta"]+ layernorm_mlp_alpha = self.hparams["layernorm_mlp_alpha"]+ layernorm_mlp_beta = self.hparams["layernorm_mlp_beta"]+ assert layernorm_full_attention_alpha == layernorm_linear_attention_alpha == layernorm_mlp_alpha+ assert layernorm_full_attention_beta == layernorm_linear_attention_beta == layernorm_mlp_beta == 1.0+ # we do not store the layernorm betas as they are all 1.0+ # layernorm alphas are stored as single residual_scale hparam+ self.gguf_writer.add_residual_scale(layernorm_full_attention_alpha)++ self.gguf_writer.add_rope_dimension_count(self.hparams["rotary_dim"])++ _experts: list[dict[str, Tensor]] | None = None++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ # process the experts separately+ if name.find("block_sparse_moe.experts") != -1:+ n_experts = self.hparams["num_local_experts"]++ assert bid is not None++ if self._experts is None:+ self._experts = [{} for _ in range(self.block_count)]++ self._experts[bid][name] = data_torch++ if len(self._experts[bid]) >= n_experts * 3:+ # merge the experts into a single 3d tensor+ for wid in ["w1", "w2", "w3"]:+ datas: list[Tensor] = []++ for xid in range(n_experts):+ ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight"+ datas.append(self._experts[bid][ename])+ del self._experts[bid][ename]++ data_torch = torch.stack(datas, dim=0)++ merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight"++ new_name = self.map_tensor_name(merged_name)++ yield from super().modify_tensors(data_torch, new_name, bid)+ return+ else:+ return++ yield from super().modify_tensors(data_torch, name, bid)@ModelBase.register("MiniMaxM2ForCausalLM")diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 98c4fa1691..a312c8aa00 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -565,6 +565,7 @@ class MODEL_ARCH(IntEnum):GROVEMOE = auto()APERTUS = auto()COGVLM = auto()+ MINIMAX01 = auto()MINIMAXM2 = auto()MINIMAXM3 = auto()RND1 = auto()@@ -1271,6 +1272,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {MODEL_ARCH.SEED_OSS: "seed_oss",MODEL_ARCH.GROVEMOE: "grovemoe",MODEL_ARCH.APERTUS: "apertus",+ MODEL_ARCH.MINIMAX01: "minimax-01",MODEL_ARCH.MINIMAXM2: "minimax-m2",MODEL_ARCH.MINIMAXM3: "minimax-m3",MODEL_ARCH.COGVLM: "cogvlm",@@ -4592,6 +4594,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_DOWN_CHEXP,MODEL_TENSOR.FFN_UP_CHEXP,],+ MODEL_ARCH.MINIMAX01: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_NORM_2,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],MODEL_ARCH.MINIMAXM2: [MODEL_TENSOR.TOKEN_EMBD,MODEL_TENSOR.OUTPUT_NORM,diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex 79d270ab8f..ee3a8d3aa4 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -225,6 +225,7 @@ class TensorNameMap:"rwkv.blocks.{bid}.ln2", # rwkv6"model.layers.{bid}.ln2", # rwkv7"model.layers.{bid}.post_attention_layernorm", # cogvlm+ "model.layers.{bid}.self_attn.norm", # minimax-01),# Attention query-key-value@@ -321,7 +322,7 @@ class TensorNameMap:"h.{bid}.self_attention.dense", # bloom"model.layers.{bid}.self_attn.o_proj", # llama-hf nemotron olmoe olmo2 phimoe"layers.{bid}.self_attn.o_proj", # embeddinggemma- "model.layers.{bid}.self_attn.out_proj", # lfm2+ "model.layers.{bid}.self_attn.out_proj", # lfm2 minimax-01"model.layers.{bid}.self_attn.linear_attn", # deci"layers.{bid}.attention.wo", # llama-pth"encoder.layer.{bid}.attention.output.dense", # bert@@ -385,6 +386,7 @@ class TensorNameMap:"model.layers.{bid}.self_attn.gate_proj", # afmoe muse-glimmer"model.layers.{bid}.linear_attn.in_proj_z", # qwen3.5"model.layers.{bid}.self_attn.g_proj", # step3.5 head-wise attention gate+ "model.layers.{bid}.self_attn.output_gate", # minimax-01),# Feed-forward normdiff --git a/models/templates/MiniMax-M1.jinja b/models/templates/MiniMax-M1.jinjanew file mode 100644index 0000000000..2d5bbf4de5--- /dev/null+++ b/models/templates/MiniMax-M1.jinja@@ -0,0 +1,91 @@+{{ '<begin_of_document>' -}}+{%- if custom_tools is defined %}+ {%- set tools = custom_tools %}+{%- endif %}+{%- if not tools is defined %}+ {%- set tools = none %}+{%- endif %}++{#- Extract system message #}+{% set ns = namespace(system_prompt='') -%}+{%- if messages[0]['role'] == 'system' %}+ {%- if messages[0]['content'] is string %}+ {%- set ns.system_prompt = messages[0]['content']|trim %}+ {%- else %}+ {%- set ns.system_prompt = messages[0]['content'][0]['text']|trim %}+ {%- endif %}+ {%- set messages = messages[1:] %}+{%- else %}+ {%- if tools is not none %}+ {%- set ns.system_prompt = "You are a helpful assistant created by Minimax based on MiniMax-M1 model." %}+ {%- else %}+ {%- set ns.system_prompt = "You are a helpful assistant created by Minimax based on MiniMax-M1 model." %}+ {%- endif %}+{%- endif %}++{#- System message #}+{%- if ns.system_prompt != '' %}+{{ '<beginning_of_sentence>system ai_setting=assistant\n' + ns.system_prompt + '<end_of_sentence>\n' -}}+{%- endif %}++{#- Tools configuration #}+{%- if tools is not none %}+{{ '<beginning_of_sentence>system tool_setting=tools\nYou are provided with these tools:\n<tools>\n' -}}+{%- for tool in tools %}+{{ tool | tojson ~ '\n' -}}+{%- endfor %}+{{ '</tools>\n\nIf you need to call tools, please respond with <tool_calls></tool_calls> XML tags, and provide tool-name and json-object of arguments, following the format below:\n<tool_calls>\n{"name": <tool-name>, "arguments": <args-json-object>}\n...\n</tool_calls><end_of_sentence>\n' -}}+{%- endif %}++{#- Process messages #}+{%- for message in messages %}+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}+ {%- if message['role'] == 'user' %}+{{ '<beginning_of_sentence>user name=user\n' -}}+{%- if message['content'] is string %}+{{ message['content']|trim -}}+{%- else %}+{%- for content in message['content'] %}+{%- if content['type'] == 'text' %}+{{ content['text']|trim -}}+{%- endif %}+{%- endfor %}+{%- endif %}+{{ '<end_of_sentence>\n' -}}+ {%- elif message['role'] == 'assistant' %}+{{ '<beginning_of_sentence>ai name=assistant\n' -}}+{%- if message['content'] is string %}+{{ message['content']|trim -}}+{%- else %}+{%- for content in message['content'] | selectattr('type', 'equalto', 'text') %}+{{ content['text']|trim -}}+{%- endfor %}+{%- endif %}+{{ '<end_of_sentence>\n' -}}+ {%- endif %}+ {%- elif 'tool_calls' in message %}+{{ '<beginning_of_sentence>ai name=assistant\n<tool_calls>\n' -}}+{%- for tool_call in message.tool_calls %}+{{ '{"name": "' + tool_call.function.name + '", "arguments": ' + tool_call.function.arguments | tojson + '}\n' -}}+{%- endfor %}+{{ '</tool_calls><end_of_sentence>\n' -}}+ {%- elif message.role == "tool" or message.role == "ipython" %}+{{ '<beginning_of_sentence>tool name=tools\n' -}}+{%- if message.content is string %}+{{ 'tool result: ' + message.content + '\n\n' -}}+{%- else %}+{%- for content in message['content'] %}+{%- if content['type'] == 'text' %}+{{ 'tool result: ' + content['text'] + '\n\n' -}}+{%- elif content.get('name') %}+{{ 'tool name: ' + content['name'] + '\ntool result: ' + content['text'] + '\n\n' -}}+{%- endif %}+{%- endfor %}+{%- endif %}+{{ '<end_of_sentence>\n' -}}+ {%- endif %}+{%- endfor %}++{%- if add_generation_prompt %}+{{ '<beginning_of_sentence>ai name=assistant\n' -}}+{%- endif %}\ No newline at end of filediff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 292ab26103..1cd9b758f3 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -128,6 +128,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_SEED_OSS, "seed_oss" },{ LLM_ARCH_GROVEMOE, "grovemoe" },{ LLM_ARCH_APERTUS, "apertus" },+ { LLM_ARCH_MINIMAX_01, "minimax-01" },{ LLM_ARCH_MINIMAX_M2, "minimax-m2" },{ LLM_ARCH_MINIMAX_M3, "minimax-m3" },{ LLM_ARCH_COGVLM, "cogvlm" },@@ -978,6 +979,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {case LLM_ARCH_QWEN35:case LLM_ARCH_QWEN35MOE:case LLM_ARCH_DEEPSEEK4:+ case LLM_ARCH_MINIMAX_01:return true;default:return false;@@ -1033,6 +1035,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {case LLM_ARCH_GRANITE_HYBRID:case LLM_ARCH_LFM2:case LLM_ARCH_LFM2MOE:+ case LLM_ARCH_MINIMAX_01:case LLM_ARCH_MINIMAX_M2:case LLM_ARCH_MINIMAX_M3:case LLM_ARCH_MISTRAL4:diff --git a/src/llama-arch.h b/src/llama-arch.hindex 18d9de186f..24252b9da9 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -153,6 +153,7 @@ enum llm_arch {LLM_ARCH_NANBEIGE,LLM_ARCH_QWEN3TTS,LLM_ARCH_POCKETTTS,+ LLM_ARCH_MINIMAX_01,LLM_ARCH_UNKNOWN,};diff --git a/src/llama-context.cpp b/src/llama-context.cppindex cd013cdb14..ce38e8a314 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -2300,6 +2300,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {model.arch == LLM_ARCH_DEEPSEEK4 ||(model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) ||model.arch == LLM_ARCH_NANBEIGE ||+ model.arch == LLM_ARCH_MINIMAX_01 ||model.arch == LLM_ARCH_MINIMAX_M3) {res = std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());} else {diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cppindex 781277f3ff..e3f0cf0ede 100644--- a/src/llama-hparams.cpp+++ b/src/llama-hparams.cpp@@ -217,6 +217,13 @@ uint32_t llama_hparams::n_embd_s() const {return n_embd_head_kda * n_embd_head_kda * n_head(); // 128 * 128 * 32 = 524288}+ if (n_embd_head_la != 0) {+ // for MiniMax-Text-01 linear attention layers+ // Full recurrent state: head_dim * head_dim * n_head+ // tensor shape for linear attention: [head_dim, head_dim, n_head]+ return n_embd_head_la * n_embd_head_la * n_head(); // 128 * 128 * 64 = 1048576+ }+// corresponds to Mamba's ssm_states sizereturn ssm_d_state * ssm_d_inner;}diff --git a/src/llama-hparams.h b/src/llama-hparams.hindex 57de808242..d9ac112b79 100644--- a/src/llama-hparams.h+++ b/src/llama-hparams.h@@ -164,6 +164,9 @@ struct llama_hparams {uint32_t ssm_dt_rank = 0;uint32_t ssm_n_group = 0;+ // for MiniMax-Text-01 linear attention+ uint32_t n_embd_head_la = 0;+// for Kimi Linear KDAuint32_t n_embd_head_kda = 0;diff --git a/src/llama-model.cpp b/src/llama-model.cppindex c810055050..3ef683d5be 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -296,6 +296,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_paramsreturn new llama_model_grovemoe(params);case LLM_ARCH_APERTUS:return new llama_model_apertus(params);+ case LLM_ARCH_MINIMAX_01:+ return new llama_model_minimax_01(params);case LLM_ARCH_MINIMAX_M2:return new llama_model_minimax_m2(params);case LLM_ARCH_MINIMAX_M3:@@ -798,6 +800,7 @@ const char * llm_type_name(llm_type type) {case LLM_TYPE_290B: return "290B";case LLM_TYPE_314B: return "314B";case LLM_TYPE_405B: return "405B";+ case LLM_TYPE_456B: return "456B";case LLM_TYPE_671B: return "671B";case LLM_TYPE_SMALL: return "0.1B";case LLM_TYPE_MEDIUM: return "0.4B";@@ -2283,7 +2286,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,filter_recr = [&](uint32_t il) {return hparams.is_recr(il) && hparams.n_ff(il) == 0;};- } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) {+ } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_MINIMAX_01) {filter_attn = [&](uint32_t il) {return il < hparams.n_layer() && !hparams.is_recr(il);};@@ -2704,6 +2707,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {case LLM_ARCH_SEED_OSS:case LLM_ARCH_GROVEMOE:case LLM_ARCH_APERTUS:+ case LLM_ARCH_MINIMAX_01:case LLM_ARCH_MINIMAX_M2:case LLM_ARCH_MINIMAX_M3:case LLM_ARCH_COGVLM:diff --git a/src/llama-model.h b/src/llama-model.hindex 341cb66fba..510ed8498b 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -99,6 +99,7 @@ enum llm_type {LLM_TYPE_290B,LLM_TYPE_314B,LLM_TYPE_405B,+ LLM_TYPE_456B,LLM_TYPE_671B,LLM_TYPE_SMALL,LLM_TYPE_MEDIUM,@@ -271,6 +272,7 @@ struct llama_layer {struct ggml_tensor * wv = nullptr;struct ggml_tensor * wo = nullptr;struct ggml_tensor * wqkv = nullptr;+ struct ggml_tensor * wg = nullptr;struct ggml_tensor * wq_a = nullptr;struct ggml_tensor * wq_b = nullptr;struct ggml_tensor * wkv_a_mqa = nullptr;diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cppnew file mode 100644index 0000000000..a6ccee1917--- /dev/null+++ b/src/models/minimax-01.cpp@@ -0,0 +1,520 @@+#include "models.h"+#include "llama-memory-recurrent.h"++void llama_model_minimax_01::load_arch_hparams(llama_model_loader & ml) {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale);++ // we use n_embd_head_la to set recurrent memory n_embd_s+ hparams.n_embd_head_la = hparams.n_embd_head_k_full;++ // Mark recurrent layers (lightning attention layers).+ if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {+ uint32_t full_attn_interval = 8;+ ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);+ for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {+ hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);+ }+ }++ switch (hparams.n_layer()) {+ case 80: type = LLM_TYPE_456B; break;+ default: type = LLM_TYPE_UNKNOWN;+ }+}++void llama_model_minimax_01::load_arch_tensors(llama_model_loader &) {+ LLAMA_LOAD_LOCALS;++ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);++ // output+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);++ // if output is NULL, init from the input tok embed+ if (output == NULL) {+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);+ }++ for (int i = 0; i < n_layer; ++i) {+ auto & layer = layers[i];++ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);++ if (!hparams.is_recr(i)) {+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);+ } else {+ layer.attn_norm_2 = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd_head_k * n_head}, 0);+ layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, 3 * n_embd_head_k * n_head}, 0);+ layer.wg = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);+ }+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);++ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);++ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, TENSOR_NOT_REQUIRED);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0);+ }+}++std::unique_ptr<llm_graph_context> llama_model_minimax_01::build_arch_graph(const llm_graph_params & params) const {+ return std::make_unique<graph>(*this, params);+}++class llm_graph_input_la : public llm_graph_input_i {+public:+ llm_graph_input_la(const llama_hparams & hparams) : hparams(hparams) {}++ void set_input(const llama_ubatch * ubatch) override {+ // this operates on assumption that we have an equal ubatch split++ const int64_t n_head = hparams.n_head();+ const int32_t n_seqs = ubatch->n_seqs;+ const int32_t n_seqs_unq = ubatch->n_seqs_unq;+ const int32_t n_tokens = ubatch->n_tokens;+ const int32_t n_seq_tokens = ubatch->n_seq_tokens;++ std::vector<llama_pos> p0(n_seqs_unq);+ std::fill(p0.begin(), p0.end(), std::numeric_limits<llama_pos>::max());++ // get lowest token position in a ubatch for each stream+ for (int i = 0; i < n_tokens; ++i) {+ llama_seq_id seq_id = ubatch->seq_id[i][0];+ int32_t seq_idx = ubatch->seq_idx[seq_id];+ llama_pos pos = ubatch->pos[i];+ if (p0[seq_idx] > pos) {+ p0[seq_idx] = pos;+ }+ }++ if (inp_slopes) {+ GGML_ASSERT(ggml_backend_buffer_is_host(inp_slopes->buffer));++ float * data = (float *) inp_slopes->data;++ float start = powf(2, -powf(2, -(log2f(n_head) - 3)));+ float ratio = start;++ for (int h = 0; h < n_head; ++h) {+ data[h] = start * powf(ratio, h);+ }+ }++ if (inp_q_decay) {+ GGML_ASSERT(ggml_backend_buffer_is_host(inp_q_decay->buffer));++ float * slopes = (float *) inp_slopes->data;+ float * data = (float *) inp_q_decay->data;++ for (int s = 0; s < n_seqs; ++s) {+ for (int i = 0; i < n_seq_tokens; ++i) {+ llama_seq_id seq_id = ubatch->seq_id[s * n_seq_tokens + i][0];+ int32_t seq_idx = ubatch->seq_idx[seq_id];+ llama_pos pos = ubatch->pos[s * n_seq_tokens + i];+ int pos_rel = pos - p0[seq_idx];++ for (int h = 0; h < n_head; ++h) {+ data[seq_idx * n_head * n_seq_tokens + i * n_head + h] = -slopes[h] * (pos_rel + 1);+ }+ }+ }+ }++ if (inp_k_decay) {+ GGML_ASSERT(ggml_backend_buffer_is_host(inp_k_decay->buffer));++ float * slopes = (float *) inp_slopes->data;+ float * data = (float *) inp_k_decay->data;++ for (int s = 0; s < n_seqs; ++s) {+ for (int i = 0; i < n_seq_tokens; ++i) {+ llama_seq_id seq_id = ubatch->seq_id[s * n_seq_tokens + i][0];+ int32_t seq_idx = ubatch->seq_idx[seq_id];+ llama_pos pos = ubatch->pos[s * n_seq_tokens + i];+ int pos_rel = pos - p0[seq_idx];++ for (int h = 0; h < n_head; ++h) {+ data[seq_idx * n_head * n_seq_tokens + i * n_head + h] = -slopes[h] * (n_seq_tokens - pos_rel - 1);+ }+ }+ }+ }++ if (inp_diag_decay) {+ GGML_ASSERT(ggml_backend_buffer_is_host(inp_diag_decay->buffer));++ float * slopes = (float *) inp_slopes->data;+ float * data = (float *) inp_diag_decay->data;++ for (int s = 0; s < n_seqs; ++s) {+ for (int h = 0; h < n_head; ++h) {+ for (int j = 0; j < n_seq_tokens; ++j) {+ llama_seq_id seq_id = ubatch->seq_id[s * n_seq_tokens + j][0];+ int32_t seq_idx = ubatch->seq_idx[seq_id];+ llama_pos pos_j = ubatch->pos[s * n_seq_tokens + j];+ int pos_rel_j = pos_j - p0[seq_idx];++ for (int i = 0; i < n_seq_tokens; ++i) {+ llama_pos pos_i = ubatch->pos[s * n_seq_tokens + i];+ int pos_rel_i = pos_i - p0[seq_idx];++ int index = pos_rel_j - pos_rel_i;+ float s_index = index >= 0 ? -slopes[h] * index : -INFINITY;+ data[seq_idx * n_head * n_seq_tokens * n_seq_tokens + h * n_seq_tokens * n_seq_tokens + j * n_seq_tokens + i] = s_index;+ }+ }+ }+ }+ }+ }++ bool can_reuse(const llm_graph_params & params) override {+ bool res = true;++ if (params.ubatch.n_seq_tokens > 1) {+ res &= ( inp_q_decay && inp_q_decay->ne[2] == params.ubatch.n_seq_tokens);+ res &= ( inp_k_decay && inp_k_decay->ne[2] == params.ubatch.n_seq_tokens);+ res &= (inp_diag_decay && inp_diag_decay->ne[1] == params.ubatch.n_seq_tokens);+ }++ return res;+ }++ const llama_hparams & hparams;++ ggml_tensor * inp_slopes = nullptr; // F32 [n_head]+ ggml_tensor * inp_q_decay = nullptr; // F32 [1, n_head, n_batch]+ ggml_tensor * inp_k_decay = nullptr; // F32 [1, n_head, n_batch]+ ggml_tensor * inp_diag_decay = nullptr; // F32 [n_batch, n_batch, n_head]+};++llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {+ const int64_t n_embd_head = hparams.n_embd_head_v();++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());+ // GGML_ASSERT(n_embd_head == n_rot); this is wrong in case of minimax, head_dim = 128, n_rot = 64++ const int64_t n_seqs = ubatch.n_seqs;+ const int64_t n_seq_tokens = ubatch.n_seq_tokens;++ GGML_ASSERT(n_seqs != 0);+ GGML_ASSERT(ubatch.equal_seqs());+ GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);++ ggml_tensor * cur;+ ggml_tensor * inpL;++ inpL = build_inp_embd(model.tok_embd);++ auto * inp_hybrid = build_inp_mem_hybrid();+ auto * inp_rs = inp_hybrid->get_recr();++ ggml_tensor * inp_pos = build_inp_pos();+ ggml_tensor * inp_out_ids = build_inp_out_ids();++ llm_graph_input_la * la = nullptr;++ auto inp = std::make_unique<llm_graph_input_la>(hparams);++ inp->inp_slopes = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head);+ ggml_set_input(inp->inp_slopes);+ cb(inp->inp_slopes, "slopes", -1);++ if (n_seq_tokens != 1) {+ inp->inp_q_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);+ ggml_set_input(inp->inp_q_decay);+ cb(inp->inp_q_decay, "q_decay_exp", -1);++ inp->inp_k_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);+ ggml_set_input(inp->inp_k_decay);+ cb(inp->inp_k_decay, "k_decay_exp", -1);++ inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs);+ ggml_set_input(inp->inp_diag_decay);+ cb(inp->inp_diag_decay, "diag_decay_exp", -1);+ }++ la = (llm_graph_input_la *) res->add_input(std::move(inp));++ ggml_tensor * slopes = la->inp_slopes;++ for (int il = 0; il < n_layer; ++il) {+ res->t_layer_inp[il] = inpL;++ ggml_tensor * inpSA = inpL;++ cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);+ cb(cur, "attn_norm", il);++ ggml_tensor * residual = cur;++ // self_attention+ if (!hparams.is_recr(il)) {+ // softmax attention layer++ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,+ n_embd_head, n_head, n_head_kv, il);++ Qcur = ggml_rope_ext(+ ctx0, Qcur, inp_pos, nullptr,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );++ Kcur = ggml_rope_ext(+ ctx0, Kcur, inp_pos, nullptr,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );++ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ cur = build_attn(inp_hybrid->get_attn(),+ model.layers[il].wo, NULL, model.layers[il].wo_s,+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);+ } else {+ // lightning attention layer++ const auto * mctx_cur = inp_rs->mctx;+ const auto kv_head = mctx_cur->get_head();++ // TODO unneeded - any way to make conv states optional in recurrent memory?+ ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);+ ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);+ ggml_build_forward_expand(gf, conv_state_all);++ float slope_scale = 1.0 - 1.0 * il / (n_layer - 1) + 1e-5;+ ggml_tensor * slope_rate = ggml_scale(ctx0, slopes, slope_scale);+ cb(slope_rate, "slope_rate", il);++ cur = ggml_reshape_4d(ctx0, cur, cur->ne[0], n_seq_tokens, 1, n_seqs);++ ggml_tensor * QKVcur = build_lora_mm(model.layers[il].wqkv, cur);+ cb(QKVcur, "QKVcur", il);++ QKVcur = ggml_silu(ctx0, QKVcur);+ cb(QKVcur, "QKVcur_silu", il);++ QKVcur = ggml_reshape_4d(ctx0, QKVcur, n_embd_head * 3, n_head, n_seq_tokens, n_seqs);++ ggml_tensor * Qcur = ggml_view_4d(ctx0, QKVcur, n_embd_head, n_head, n_seq_tokens, n_seqs, QKVcur->nb[1], QKVcur->nb[2], QKVcur->nb[3], 0*ggml_element_size(QKVcur)*n_embd_head);+ ggml_tensor * Kcur = ggml_view_4d(ctx0, QKVcur, n_embd_head, n_head, n_seq_tokens, n_seqs, QKVcur->nb[1], QKVcur->nb[2], QKVcur->nb[3], 1*ggml_element_size(QKVcur)*n_embd_head);+ ggml_tensor * Vcur = ggml_view_4d(ctx0, QKVcur, n_embd_head, n_head, n_seq_tokens, n_seqs, QKVcur->nb[1], QKVcur->nb[2], QKVcur->nb[3], 2*ggml_element_size(QKVcur)*n_embd_head);++ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ // get previous KV+ ggml_tensor * la_states_all = mctx_cur->get_s_l(il);+ ggml_tensor * state = build_rs(inp_rs, la_states_all, hparams.n_embd_s(), n_seqs);++ ggml_tensor * kv_old = ggml_reshape_4d(ctx0, state, n_embd_head, n_embd_head, n_head, n_seqs);+ cb(kv_old, "kv_old", il);++ ggml_tensor * qkv = nullptr;+ ggml_tensor * kv_new = nullptr;++ if (n_seq_tokens == 1) {+ // lightning attention - optimized single token case for TG++ ggml_tensor * slopes_neg = ggml_scale(ctx0, slope_rate, -1.0);+ cb(slopes_neg, "slopes_neg", il);++ ggml_tensor * ratio = ggml_exp(ctx0, slopes_neg);+ cb(ratio, "ratio", il);++ ggml_tensor * ratio_3d = ggml_reshape_3d(ctx0, ratio, 1, 1, n_head);+ cb(ratio_3d, "ratio3d", il);++ ggml_tensor * v_trans = ggml_cont(ctx0, ggml_permute(ctx0, Vcur, 1, 2, 0, 3));+ cb(v_trans, "v_trans", il);++ ggml_tensor * k_trans = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 1, 2, 0, 3));+ cb(k_trans, "k_trans", il);++ ggml_tensor * kv_cur = ggml_mul_mat(ctx0, k_trans, v_trans);+ cb(kv_cur, "kv_cur", il);++ ggml_tensor * kv_old_s = ggml_mul(ctx0, kv_old, ratio_3d);+ cb(kv_old_s, "kv_old_s", il);++ kv_new = ggml_add(ctx0, kv_old_s, kv_cur);+ cb(kv_new, "kv_new", il);++ ggml_tensor * q_trans = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);+ cb(q_trans, "q_trans", il);++ qkv = ggml_mul_mat(ctx0, kv_new, q_trans);+ cb(qkv, "qkv", il);+ } else if(n_seq_tokens > 1) {+ // lightning attention - general multi token case for PP++ ggml_tensor * q_decay_exp = la->inp_q_decay;+ ggml_tensor * k_decay_exp = la->inp_k_decay;+ ggml_tensor * diag_decay_exp = la->inp_diag_decay;++ ggml_tensor * q_decay = ggml_exp(ctx0, ggml_scale(ctx0, q_decay_exp, slope_scale));+ cb(q_decay, "q_decay", il);+ ggml_tensor * k_decay = ggml_exp(ctx0, ggml_scale(ctx0, k_decay_exp, slope_scale));+ cb(k_decay, "k_decay", il);+ ggml_tensor * diag_decay = ggml_exp(ctx0, ggml_scale(ctx0, diag_decay_exp, slope_scale));+ cb(diag_decay, "diag_decay", il);++ ggml_tensor * q_s = ggml_mul(ctx0, Qcur, q_decay);+ cb(q_s, "q_s", il);++ ggml_tensor * q_s_trans = ggml_permute(ctx0, q_s, 0, 2, 1, 3);+ cb(q_s_trans, "q_s_trans", il);++ ggml_tensor * qkv_none_diag = ggml_mul_mat(ctx0, kv_old, q_s_trans);+ cb(qkv_none_diag, "qkv_none_diag", il);++ ggml_tensor * q_trans = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);+ cb(q_trans, "q_trans", il);++ ggml_tensor * k_trans = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);+ cb(k_trans, "k_trans", il);++ ggml_tensor * qk = ggml_mul_mat(ctx0, k_trans, q_trans);+ cb(qk, "qk", il);++ qk = ggml_mul(ctx0, qk, diag_decay);+ cb(qk, "qk_s", il);++ ggml_tensor * v_trans = ggml_cont(ctx0, ggml_permute(ctx0, Vcur, 1, 2, 0, 3));+ cb(v_trans, "v_trans", il);++ ggml_tensor * qkv_diag = ggml_mul_mat(ctx0, v_trans, qk);+ cb(qkv_diag, "qkv_diag", il);++ qkv = ggml_add(ctx0, qkv_none_diag, qkv_diag);+ cb(qkv, "qkv", il);++ ggml_build_forward_expand(gf, qkv);++ ggml_tensor * slopes_neg = ggml_scale(ctx0, slope_rate, -1.0*n_seq_tokens);+ cb(slopes_neg, "slopes_neg", il);++ ggml_tensor * block_decay = ggml_exp(ctx0, slopes_neg);+ cb(block_decay, "block_decay", il);++ ggml_tensor * block_decay_3d = ggml_reshape_3d(ctx0, block_decay, 1, 1, n_head);+ cb(block_decay_3d, "block_decay_3d", il);++ ggml_tensor * kv_old_s = ggml_mul(ctx0, kv_old, block_decay_3d);+ cb(kv_old_s, "kv_old_s", il);++ ggml_tensor * k_after_decay = ggml_mul(ctx0, Kcur, k_decay);+ cb(k_after_decay, "k_after_decay", il);++ ggml_tensor * k_after_decay_trans = ggml_cont(ctx0, ggml_permute(ctx0, k_after_decay, 1, 2, 0, 3));+ cb(k_after_decay_trans, "k_after_decay_trans", il);++ ggml_tensor * kv_cur = ggml_mul_mat(ctx0, k_after_decay_trans, v_trans);+ cb(kv_cur, "kv_cur", il);++ kv_new = ggml_add(ctx0, kv_old_s, kv_cur);+ cb(kv_new, "kv_new", il);+ }++ // store new KV+ ggml_build_forward_expand(gf,+ ggml_cpy(ctx0, kv_new,+ ggml_view_1d(ctx0, la_states_all, hparams.n_embd_s() * n_seqs,+ kv_head * hparams.n_embd_s() * ggml_element_size(la_states_all))));++ qkv = ggml_cont(ctx0, ggml_permute(ctx0, qkv, 0, 2, 1, 3));+ cb(qkv, "qkv_permuted", il);++ qkv = ggml_reshape_4d(ctx0, qkv, qkv->ne[0]*qkv->ne[1], qkv->ne[2], 1, qkv->ne[3]);++ // norm+ ggml_tensor * qkv_norm = build_norm(qkv,+ model.layers[il].attn_norm_2, NULL,+ LLM_NORM_RMS, il);+ cb(qkv_norm, "qkv_norm", il);++ ggml_tensor * g = build_lora_mm(model.layers[il].wg, cur);+ cb(g, "g", il);++ g = ggml_sigmoid(ctx0, g);+ cb(g, "g_sigm", il);++ cur = ggml_mul(ctx0, g, qkv_norm);++ cur = build_lora_mm(model.layers[il].wo, cur);+ cb(cur, "attn_out", il);++ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens*n_seqs);+ cb(cur, "attn_out", il);+ }++ if (il == n_layer - 1 && inp_out_ids) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ residual = ggml_get_rows(ctx0, residual, inp_out_ids);+ }++ residual = ggml_scale(ctx0, residual, hparams.f_residual_scale);+ cb(residual, "residual_scaled_attn", il);++ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, residual);+ cb(ffn_inp, "ffn_inp", il);++ // MoE branch+ cur = build_norm(ffn_inp,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, il);+ cb(cur, "ffn_norm", il);++ residual = cur;++ cur = build_moe_ffn(cur,+ model.layers[il].ffn_gate_inp,+ model.layers[il].ffn_up_exps,+ model.layers[il].ffn_gate_exps,+ model.layers[il].ffn_down_exps,+ model.layers[il].ffn_exp_probs_b,+ n_expert, n_expert_used,+ LLM_FFN_SILU, true,+ hparams.expert_weights_scale,+ LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,+ il);+ cb(cur, "ffn_moe_out", il);++ residual = ggml_scale(ctx0, residual, hparams.f_residual_scale);+ cb(residual, "residual_scaled_ffn", il);++ cur = ggml_add(ctx0, cur, residual);+ cb(cur, "ffn_out", il);++ cur = build_cvec(cur, il);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = build_norm(cur,+ model.output_norm, NULL,+ LLM_NORM_RMS, -1);++ cb(cur, "result_norm", -1);+ res->t_embd = cur;++ // lm_head+ cur = build_lora_mm(model.output, cur, model.output_s);++ cb(cur, "result_output", -1);+ res->t_logits = cur;++ ggml_build_forward_expand(gf, cur);+}diff --git a/src/models/models.h b/src/models/models.hindex ddb9ae2f12..cefaf56eb1 100644--- a/src/models/models.h+++ b/src/models/models.h@@ -2043,6 +2043,19 @@ struct llama_model_apertus : public llama_model_base {};+struct llama_model_minimax_01 : public llama_model_base {+ llama_model_minimax_01(const struct llama_model_params & params) : llama_model_base(params) {}+ void load_arch_hparams(llama_model_loader & ml) override;+ void load_arch_tensors(llama_model_loader & ml) override;++ struct graph : public llm_graph_context {+ graph(const llama_model & model, const llm_graph_params & params);+ };++ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;+};++struct llama_model_minimax_m2 : public llama_model_base {llama_model_minimax_m2(const struct llama_model_params & params) : llama_model_base(params) {}void load_arch_hparams(llama_model_loader & ml) override;diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex e900bdc0da..f28be754fd 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -243,6 +243,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));+ ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);for (uint32_t il = 0; il < n_layer; il++) {ggml_tensor t;@@ -364,6 +365,7 @@ static bool moe_mandatory(const llm_arch arch) {case LLM_ARCH_SMALLTHINKER:case LLM_ARCH_LLADA_MOE:case LLM_ARCH_GROVEMOE:+ case LLM_ARCH_MINIMAX_01:case LLM_ARCH_MINIMAX_M2:case LLM_ARCH_MINIMAX_M3:case LLM_ARCH_RND1:@@ -436,7 +438,7 @@ static bool arch_supported(const llm_arch arch) {// FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.#ifdef GGML_USE_WEBGPU- if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA) {+ if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MINIMAX_01) {return false;}#endif // GGML_USE_WEBGPU