LLM inference in C/C++
C++ ggml
Keep Minimax's indexer tensors at F32 for speed and accuracy (#26144)
Bartowski7657a6c26a7c
diff --git a/conversion/minimax.py b/conversion/minimax.pyindex cbbdfe3ae8..e82e393a38 100644--- a/conversion/minimax.py+++ b/conversion/minimax.py@@ -58,6 +58,11 @@ class MiniMaxM2Model(TextModel):class MiniMaxM3Model(MiniMaxM2Model):model_arch = gguf.MODEL_ARCH.MINIMAXM3+ def tensor_force_quant(self, name, new_name, bid, n_dims):+ if ".indexer." in new_name:+ return gguf.GGMLQuantizationType.F32+ return super().tensor_force_quant(name, new_name, bid, n_dims)+def set_gguf_parameters(self):super().set_gguf_parameters()diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex caf7733a5b..7c0bac07d0 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -326,6 +326,10 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * paramquantize &= name.find("ssm_conv1d") == std::string::npos;quantize &= name.find("shortconv.conv.weight") == std::string::npos;+ // do not quantize MiniMax's indexer projection weights, they are tiny+ quantize &= name.find("indexer.k_proj.weight") == std::string::npos;+ quantize &= name.find("indexer.q_proj.weight") == std::string::npos;+// do not quantize RWKV's small yet 2D weightsquantize &= name.find("time_mix_first.weight") == std::string::npos;quantize &= name.find("time_mix_w0.weight") == std::string::npos;