LLM inference in C/C++
C++ ggml
TP: fix Phi3, Bert, Plamo2/3, ChatGLM (#25536)
Johannes Gäßler2e1fd76490af
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex eaf3f35d2d..4c10e4126f 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -360,8 +360,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const strstatic const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");- static const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");- static const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");+ static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight");+ static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias");+ static const std::regex pattern_ffn_gate_weight ("blk\\.\\d*\\.ffn_gate(_exps)?.weight");+ static const std::regex pattern_ffn_gate_bias ("blk\\.\\d*\\.ffn_gate(_exps)?.bias");static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");@@ -469,10 +471,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str}// FFN- if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight)) {+ if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_gate_weight)) {return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");}- if (std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) {+ if (std::regex_match(tensor_name, pattern_ffn_up_bias) || std::regex_match(tensor_name, pattern_ffn_gate_bias)) {return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");}if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {@@ -556,6 +558,14 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const strGGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);return {{n_embd, 1}, {n_embd_gqa, 2}};}+ if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias)) {+ const int64_t n_ff = hparams.n_ff(il);+ // some models such as Phi 3 have fused up + gate tensors named "up" tensors, which need to be segmented+ if (tensor->ne[axis] == 2*n_ff) {+ return {{n_ff, 2}};+ }+ return {{tensor->ne[axis], 1}};+ }if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {const int64_t n_ff_exp = hparams.n_ff_exp;GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);@@ -632,7 +642,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str}// FFN- if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || std::regex_match(tensor_name, pattern_ffn_up_gate_bias) ||+ if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) ||+ std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) ||std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) {const int64_t blck_size_perf = std::lcm(blck_size, 128);GGML_ASSERT(segments.size() == 1);diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex 796d490749..57d33a627d 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -40,8 +40,10 @@ static double nmse(const std::vector<float> & a, const std::vector<float> & b) {}static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {+ size_t seed = *(const size_t *) userdata;std::hash<std::string> hasher;- std::mt19937 gen(hasher(tensor->name) + *(const size_t *) userdata);+ seed ^= hasher(tensor->name);+ std::mt19937 gen(seed);std::normal_distribution<float> dis(0.0f, 1.0e-2f);const int64_t ne = ggml_nelements(tensor);