Skip to content

LLM inference in C/C++

C++ ggml

metal : fix null-pipeline crash for F16 src1 mul_mat/mul_mat_id (#25648)

QuintinShawd882575cc8a6
diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp
index 2eb9820bff..6f1638a114 100644
--- a/ggml/src/ggml-metal/ggml-metal-common.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-common.cpp
@@ -1,10 +1,27 @@
#include "ggml-metal-common.h"
+#include "ggml.h"
#include "ggml-impl.h"
#include "ggml-backend-impl.h"
#include <vector>
+bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {
+ const int64_t ne00 = op->src[0]->ne[0];
+ const int64_t ne11 = op->src[1]->ne[1];
+
+ return !ggml_is_transposed(op->src[0]) &&
+ !ggml_is_transposed(op->src[1]) &&
+ has_simdgroup_mm && ne00 >= 64 && ne11 > 8;
+}
+
+bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {
+ const int64_t ne00 = op->src[0]->ne[0];
+ const int64_t ne21 = op->src[2]->ne[1];
+
+ return has_simdgroup_mm && ne00 >= 64 && ne21 >= 32;
+}
+
// represents a memory range (i.e. an interval from a starting address p0 to an ending address p1 in a given buffer pb)
// the type indicates whether it is a source range (i.e. ops read data from it) or a destination range (i.e. ops write data to it)
struct ggml_mem_range {
diff --git a/ggml/src/ggml-metal/ggml-metal-common.h b/ggml/src/ggml-metal/ggml-metal-common.h
index 3acbc6ae17..66abdb52ef 100644
--- a/ggml/src/ggml-metal/ggml-metal-common.h
+++ b/ggml/src/ggml-metal/ggml-metal-common.h
@@ -47,6 +47,10 @@ bool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const struct ggml_tensor * ten
// if it proves to work well, we can start using it for other backends in the future
void ggml_graph_optimize(struct ggml_cgraph * gf);
+// mat-mat vs mat-vec dispatch; used by both supports_op and ggml_metal_op_mul_mat*
+bool ggml_metal_op_mul_mat_use_mm (const struct ggml_tensor * op, bool has_simdgroup_mm);
+bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm);
+
#ifdef __cplusplus
}
#endif
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 85c0f576b9..a053887a33 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -3,6 +3,7 @@
#import "ggml-impl.h"
#import "ggml-backend-impl.h"
#import "ggml-metal-impl.h"
+#import "ggml-metal-common.h"
#include <Foundation/Foundation.h>
@@ -788,6 +789,10 @@ void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {
}
void ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline) {
+ if (!pipeline.pipeline) {
+ GGML_ABORT("%s: nil Metal pipeline (missing kernel; see compile_pipeline log above)\n", __func__);
+ }
+
[encoder->obj setComputePipelineState:pipeline.pipeline->obj];
}
@@ -1410,6 +1415,30 @@ void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t
}
}
+static bool ggml_metal_supports_mul_mat_op(
+ bool has_simdgroup_reduction,
+ const struct ggml_tensor * op,
+ bool src0_f16_has_mv,
+ bool mm_path) {
+ if (!has_simdgroup_reduction || op->src[0]->type == GGML_TYPE_NVFP4) {
+ return false;
+ }
+
+ if (op->src[1]->type != GGML_TYPE_F16) {
+ return true;
+ }
+
+ if (op->src[0]->type == GGML_TYPE_BF16) {
+ return false;
+ }
+
+ if (src0_f16_has_mv && op->src[0]->type == GGML_TYPE_F16) {
+ return true;
+ }
+
+ return mm_path;
+}
+
bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op) {
const bool has_simdgroup_mm = dev->props.has_simdgroup_mm;
const bool has_simdgroup_reduction = dev->props.has_simdgroup_reduction;
@@ -1713,9 +1742,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_OP_GATED_DELTA_NET:
return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0;
case GGML_OP_SOLVE_TRI:
+ return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_MUL_MAT:
+ return ggml_metal_supports_mul_mat_op(
+ has_simdgroup_reduction, op, true,
+ ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm));
case GGML_OP_MUL_MAT_ID:
- return has_simdgroup_reduction && op->src[0]->type != GGML_TYPE_NVFP4;
+ return ggml_metal_supports_mul_mat_op(
+ has_simdgroup_reduction, op, false,
+ ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm));
case GGML_OP_SET:
case GGML_OP_CPY:
case GGML_OP_DUP:
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 89c8483b37..7671d1d015 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -2362,10 +2362,6 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
const int16_t r2 = ne12/ne02;
const int16_t r3 = ne13/ne03;
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
- // to the matrix-vector kernel
- const int ne11_mm_min = 8;
-
// first try to use small-batch mat-mv kernels
// these should be efficient for BS [2, ~8]
if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) &&
@@ -2468,12 +2464,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + r0ptg - 1)/r0ptg), ((ne11 + r1ptg - 1)/r1ptg), ne12*ne13, 32, nsg, 1);
- } else if (
- !ggml_is_transposed(op->src[0]) &&
- !ggml_is_transposed(op->src[1]) &&
- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
- props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) {
+ } else if (ggml_metal_op_mul_mat_use_mm(op, props_dev->has_simdgroup_mm)) {
//GGML_LOG_INFO("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
// some Metal matrix data types require aligned pointers
@@ -2622,13 +2613,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
const uint32_t r2 = 1;
const uint32_t r3 = 1;
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
- // to the matrix-vector kernel
- // ne20 = n_used_experts
- // ne21 = n_rows (batch size)
- const int ne21_mm_id_min = 32;
-
- if (props_dev->has_simdgroup_mm && ne00 >= 64 && (ne21 >= ne21_mm_id_min)) {
+ if (ggml_metal_op_mul_mat_id_use_mm(op, props_dev->has_simdgroup_mm)) {
// some Metal matrix data types require aligned pointers
// ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)
//switch (op->src[0]->type) {