Skip to content

LLM inference in C/C++

C++ ggml

metal : clamp K extent in tensor API mat-mat kernel for K not a multiple of 32 (#27450)

Georgi Gerganova30273376ef6
diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal
index 949931c8dc..27f97b5e07 100644
--- a/ggml/src/ggml-metal/ggml-metal.metal
+++ b/ggml/src/ggml-metal/ggml-metal.metal
@@ -10365,9 +10365,12 @@ kernel void kernel_mul_mm(
auto tB = tensor(ptrB, dextents<int32_t, 2>(K, N), array<int, 2>({1, strideB}));
// Configure matmul operation
+ // note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static
+ // N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0
+ // ref: https://github.com/ggml-org/llama.cpp/pull/27064
mpp::tensor_ops::matmul2d<
mpp::tensor_ops::matmul2d_descriptor(
- NRB, NRA, N_MM_NK_TOTAL, false, true, true,
+ NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
@@ -10419,10 +10422,14 @@ kernel void kernel_mul_mm(
threadgroup_barrier(mem_flags::mem_threadgroup);
// === PHASE 2: Tensor matmul ===
- auto mA = tA.slice(0, 0);
- auto mB = tB.slice(loop_k, rb);
+ // Clamp the K extent of both operand tensors to the remaining valid K range so
+ // the dynamic-K op never reads past the K extent of src1 (or the staged A tile).
+ const int kExt = min(N_MM_NK_TOTAL, K - loop_k);
- mm.run(mB, mA, cT);
+ auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, N_MM_NK_TOTAL}));
+ auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents<int32_t, 2>(kExt, N - rb), array<int, 2>({1, strideB}));
+
+ mm.run(tBv, tAv, cT);
threadgroup_barrier(mem_flags::mem_threadgroup);
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 17098825bc..89b954a7d1 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9298,6 +9298,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1}));
+ // K not a multiple of 32
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 65, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 32, 80, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 588, {1, 1}, {1, 1})); // 14*14*3, e.g. conv_2d im2col
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 80, {4, 1}, {1, 1}));
+
#if 0
// test the mat-mat path for Metal
for (int k = 1; k < 512; ++k) {