LLM inference in C/C++
C++ ggml
mamba2: remove hardcoded 2x expansion factor and invalid d_inner % d_state check (#23082)
Arsen Arutunan960d628f46d1
diff --git a/conversion/mamba.py b/conversion/mamba.pyindex be0e36a29b..43d559ffb0 100644--- a/conversion/mamba.py+++ b/conversion/mamba.py@@ -114,7 +114,8 @@ class Mamba2Model(TextModel):hparams["text_config"] = hparams["llm_config"]super().__init__(dir_model, *args, hparams=hparams, **kwargs)self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"])- self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or 2 * self.d_model+ self.expand = self.find_hparam(["mamba_expand", "expand"], optional=True) or 2+ self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or self.expand * self.d_modelself.n_group = self.find_hparam(["n_groups"], optional=True) or 1def set_vocab(self):@@ -144,11 +145,9 @@ class Mamba2Model(TextModel):rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5- # Fail early for models which don't have a block expansion factor of 2- # TODO: does this really matter?# skip the assertion for FalconH1 Modelif self.model_arch != gguf.MODEL_ARCH.FALCON_H1:- assert self.d_inner == 2 * self.d_model+ assert self.d_inner == self.expand * self.d_modelassert self.d_inner % head_dim == 0self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the defaultdiff --git a/src/models/mamba-base.cpp b/src/models/mamba-base.cppindex c37f29c487..fd3fe3f032 100644--- a/src/models/mamba-base.cpp+++ b/src/models/mamba-base.cpp@@ -169,7 +169,6 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,GGML_ASSERT(ubatch.equal_seqs());GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);GGML_ASSERT(d_inner % n_head == 0);- GGML_ASSERT(d_inner % d_state == 0);GGML_ASSERT(d_inner % n_group == 0);ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);diff --git a/src/models/mamba2.cpp b/src/models/mamba2.cppindex c5951cf0f7..d5c167cf05 100644--- a/src/models/mamba2.cpp+++ b/src/models/mamba2.cpp@@ -39,10 +39,11 @@ void llama_model_mamba2::load_arch_tensors(llama_model_loader &) {const int64_t d_inner = hparams.ssm_d_inner;const int64_t d_state = hparams.ssm_d_state;const int64_t n_group = hparams.ssm_n_group;- const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_head;+ const int64_t dt_rank = hparams.ssm_dt_rank;++ const int64_t conv_dim = d_inner + 2 * n_group * d_state;+ const int64_t d_in_proj = d_inner + conv_dim + dt_rank;- // only an expansion factor of 2 is supported for now- GGML_ASSERT(2 * n_embd == d_inner);tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);@@ -68,11 +69,11 @@ void llama_model_mamba2::load_arch_tensors(llama_model_loader &) {layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, d_inner + 2*n_group*d_state}, 0);layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {d_inner + 2*n_group*d_state}, 0);- layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {n_head}, 0);+ layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {dt_rank}, 0);// no "weight" suffix for these- layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0);- layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, n_head}, 0);+ layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, dt_rank}, 0);+ layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, dt_rank}, 0);layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {d_inner / n_group, n_group}, 0);