LLM inference in C/C++
C++ ggml
metal : clamp K extent in tensor API mat-mat kernel for K not a multiple of 32 (#27450)
Georgi Gerganova30273376ef6
diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metalindex 949931c8dc..27f97b5e07 100644--- a/ggml/src/ggml-metal/ggml-metal.metal+++ b/ggml/src/ggml-metal/ggml-metal.metal@@ -10365,9 +10365,12 @@ kernel void kernel_mul_mm(auto tB = tensor(ptrB, dextents<int32_t, 2>(K, N), array<int, 2>({1, strideB}));// Configure matmul operation+ // note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static+ // N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0+ // ref: https://github.com/ggml-org/llama.cpp/pull/27064mpp::tensor_ops::matmul2d<mpp::tensor_ops::matmul2d_descriptor(- NRB, NRA, N_MM_NK_TOTAL, false, true, true,+ NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;@@ -10419,10 +10422,14 @@ kernel void kernel_mul_mm(threadgroup_barrier(mem_flags::mem_threadgroup);// === PHASE 2: Tensor matmul ===- auto mA = tA.slice(0, 0);- auto mB = tB.slice(loop_k, rb);+ // Clamp the K extent of both operand tensors to the remaining valid K range so+ // the dynamic-K op never reads past the K extent of src1 (or the staged A tile).+ const int kExt = min(N_MM_NK_TOTAL, K - loop_k);- mm.run(mB, mA, cT);+ auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, N_MM_NK_TOTAL}));+ auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents<int32_t, 2>(kExt, N - rb), array<int, 2>({1, strideB}));++ mm.run(tBv, tAv, cT);threadgroup_barrier(mem_flags::mem_threadgroup);}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 17098825bc..89b954a7d1 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9298,6 +9298,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1}));+ // K not a multiple of 32+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 65, {1, 1}, {1, 1}));+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {1, 1}, {1, 1}));+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1}));+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1}));+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 588, {1, 1}, {1, 1})); // 14*14*3, e.g. conv_2d im2col+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {4, 1}, {1, 1}));+#if 0// test the mat-mat path for Metalfor (int k = 1; k < 512; ++k) {