LLM inference in C/C++
C++ ggml
TP: enable tensor split for LFM2/LFM2MOE (#26993)
Chris Danisb2e5e9b28b24
diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 4089544016..c9b504c333 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -1062,8 +1062,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {case LLM_ARCH_NEMOTRON_H:case LLM_ARCH_NEMOTRON_H_MOE:case LLM_ARCH_GRANITE_HYBRID:- case LLM_ARCH_LFM2:- case LLM_ARCH_LFM2MOE:case LLM_ARCH_MINIMAX_01:case LLM_ARCH_MINIMAX_M2:case LLM_ARCH_MINIMAX_M3:diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 3759c86259..d7874e0a92 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -487,6 +487,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const strreturn get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");}if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {+ if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) {+ // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, "");+ }return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");}if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {