Skip to content

LLM inference in C/C++

C++ ggml

metal: fix metal tensor kernel_mul_mm error for large tensor (#28748)

SecondSeed48e985756788
diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal
index dee13d8786..dfa5102c56 100644
--- a/ggml/src/ggml-metal/kernels/mul_mm.metal
+++ b/ggml/src/ggml-metal/kernels/mul_mm.metal
@@ -135,11 +135,15 @@ kernel void kernel_mul_mm(
}
// Store result tile to output matrix (with batch offset)
- // cT.store handles bounds checking via tD's extents (M, N)
- device float * dstBatch = (device float *)dst + im * N * M;
+ // a slice is not used because its byte offset is int32 and wraps at 2 GiB
+ // ref: https://github.com/ggml-org/llama.cpp/pull/28748
+ const int tileM = min(NRA, M - ra);
+ const int tileN = min(NRB, N - rb);
- auto tD = tensor(dstBatch, dextents<int32_t, 2>(M, N), array<int, 2>({1, M}));
- cT.store(tD.slice(ra, rb));
+ device float * dstTile = (device float *)dst + (uint64_t) im * N * M + (uint64_t) rb * M + ra;
+
+ auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, M}));
+ cT.store(tD);
}
#else