LLM inference in C/C++
C++ ggml
metal : assert shared memory padding (#27951)
Georgi Gerganov3173a56471c1
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex a82caa5e43..4e855be446 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -593,7 +593,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan(ggml_me// - sgptg floats for shared_x_dt (nsg)// - sgptg floats for shared_dA (nsg)// Total: nsg * (32 + 2) floats- res.smem = (32 + 2)*sizeof(float)*nsg;+ res.smem = GGML_PAD((32 + 2)*sizeof(float)*nsg, 16);return res;}@@ -1029,6 +1029,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(g}res.smem = (size_t) ne02*ne20*sizeof(uint16_t);+ res.smem = GGML_PAD(res.smem, 16);return res;}diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 41ce90dc8a..85c0f576b9 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -800,6 +800,9 @@ void ggml_metal_encoder_set_buffer(ggml_metal_encoder_t encoder, struct ggml_met}void ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder, size_t size, int idx) {+ // ref: https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setthreadgroupmemorylength(_:index:)+ GGML_ASSERT(size % 16 == 0);+[encoder->obj setThreadgroupMemoryLength:size atIndex:idx];}diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex f6f2fdc86c..89c8483b37 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -948,7 +948,7 @@ int ggml_metal_op_sum(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);- ggml_metal_encoder_set_threadgroup_memory_size(enc, nsg * sizeof(float), 0);+ ggml_metal_encoder_set_threadgroup_memory_size(enc, GGML_PAD(nsg * sizeof(float), 16), 0);ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, nth, 1, 1);