LLM inference in C/C++
C++ ggml
TP: fix granularity for Qwen 3.5/3.6 + 3 GPUs (#23843)
Johannes Gäßler8b0e0db60645
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex a8323c8fb1..914fc423b1 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -410,16 +410,16 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const strauto get_tensor_config = [&]() -> tensor_config {// standard attentionif (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {- return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");}if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) {- return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");}if (std::regex_match(tensor_name, pattern_qkv_weight)) {- return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");}if ( std::regex_match(tensor_name, pattern_qkv_bias)) {- return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");}if (std::regex_match(tensor_name, pattern_qk_norm)) {return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");@@ -435,7 +435,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str}if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {- return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");}if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a)) {return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");