LLM inference in C/C++
C++ ggml
llama-quant : exclude i32 ffn_gate_tid2eid routing table from quantization (#25787)
Yash Raj Pandey4937ca83f4f3
diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex b66759b277..b187f5de96 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -306,6 +306,9 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param// NOTE: can't use LLM_TN here because the layer number is not knownquantize &= name.find("ffn_gate_inp.weight") == std::string::npos;+ // do not quantize the i32 token-id -> expert-id routing table (DeepSeek-V4)+ quantize &= name.find("ffn_gate_tid2eid.weight") == std::string::npos;+// these are very small (e.g. 4x4)quantize &= name.find("altup") == std::string::npos;quantize &= name.find("laurel") == std::string::npos;