Skip to content

LLM inference in C/C++

C++ ggml

metal : avoid `threadgroup` matrix array instantiation in kernel_lightning_indexer (#26646)

JamePenge40bf886420d
diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal
index 7d12cb0fe3..92258b7374 100644
--- a/ggml/src/ggml-metal/ggml-metal.metal
+++ b/ggml/src/ggml-metal/ggml-metal.metal
@@ -11328,8 +11328,8 @@ kernel void kernel_lightning_indexer(
const int i_kv_0 = tgpig.x*NK; // first key of this threadgroup
const int i_kv = i_kv_0 + sgitg*NKPSG; // first key of this simdgroup
- threadgroup half4x4 sk4x4[NK*DK16];
- threadgroup half * sk = (threadgroup half *) sk4x4;
+ threadgroup half sk[NK * DK16 * 16];
+ threadgroup half4x4 * sk4x4 = (threadgroup half4x4 *) sk;
for (short i = tiitg; i < NK*DK16; i += NTG) {
const short ik = i/DK16;