Skip to content

LLM inference in C/C++

C++ ggml

metal : fix NORM/RMS_NORM for row lengths that leave a partial simdgroup (#26708)

robertomeronia194a75b7eec
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index c5d7619c12..6d324056dd 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -3816,7 +3816,7 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {
}
nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
- nth = std::min(nth, args.ne00_t);
+ nth = std::min(nth, (args.ne00_t + 31)/32*32);
const size_t smem = pipeline.smem;
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index fbbfee6302..6d474bc114 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -8722,6 +8722,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true));
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, true));
}
+ // row lengths that are not a multiple of 32, for the scalar (33) and float4 (132, 260) paths
+ for (uint32_t n : { 33, 132, 260 }) {
+ for (bool v : { false, true }) {
+ test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps));
+ test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps));
+ }
+ }
}
// in-place tests