LLM inference in C/C++
C++ ggml
metal : fix null-pipeline crash for F16 src1 mul_mat/mul_mat_id (#25648)
QuintinShawd882575cc8a6
diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cppindex 2eb9820bff..6f1638a114 100644--- a/ggml/src/ggml-metal/ggml-metal-common.cpp+++ b/ggml/src/ggml-metal/ggml-metal-common.cpp@@ -1,10 +1,27 @@#include "ggml-metal-common.h"+#include "ggml.h"#include "ggml-impl.h"#include "ggml-backend-impl.h"#include <vector>+bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {+ const int64_t ne00 = op->src[0]->ne[0];+ const int64_t ne11 = op->src[1]->ne[1];++ return !ggml_is_transposed(op->src[0]) &&+ !ggml_is_transposed(op->src[1]) &&+ has_simdgroup_mm && ne00 >= 64 && ne11 > 8;+}++bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {+ const int64_t ne00 = op->src[0]->ne[0];+ const int64_t ne21 = op->src[2]->ne[1];++ return has_simdgroup_mm && ne00 >= 64 && ne21 >= 32;+}+// represents a memory range (i.e. an interval from a starting address p0 to an ending address p1 in a given buffer pb)// the type indicates whether it is a source range (i.e. ops read data from it) or a destination range (i.e. ops write data to it)struct ggml_mem_range {diff --git a/ggml/src/ggml-metal/ggml-metal-common.h b/ggml/src/ggml-metal/ggml-metal-common.hindex 3acbc6ae17..66abdb52ef 100644--- a/ggml/src/ggml-metal/ggml-metal-common.h+++ b/ggml/src/ggml-metal/ggml-metal-common.h@@ -47,6 +47,10 @@ bool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const struct ggml_tensor * ten// if it proves to work well, we can start using it for other backends in the futurevoid ggml_graph_optimize(struct ggml_cgraph * gf);+// mat-mat vs mat-vec dispatch; used by both supports_op and ggml_metal_op_mul_mat*+bool ggml_metal_op_mul_mat_use_mm (const struct ggml_tensor * op, bool has_simdgroup_mm);+bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm);+#ifdef __cplusplus}#endifdiff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 85c0f576b9..a053887a33 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -3,6 +3,7 @@#import "ggml-impl.h"#import "ggml-backend-impl.h"#import "ggml-metal-impl.h"+#import "ggml-metal-common.h"#include <Foundation/Foundation.h>@@ -788,6 +789,10 @@ void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {}void ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline) {+ if (!pipeline.pipeline) {+ GGML_ABORT("%s: nil Metal pipeline (missing kernel; see compile_pipeline log above)\n", __func__);+ }+[encoder->obj setComputePipelineState:pipeline.pipeline->obj];}@@ -1410,6 +1415,30 @@ void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t}}+static bool ggml_metal_supports_mul_mat_op(+ bool has_simdgroup_reduction,+ const struct ggml_tensor * op,+ bool src0_f16_has_mv,+ bool mm_path) {+ if (!has_simdgroup_reduction || op->src[0]->type == GGML_TYPE_NVFP4) {+ return false;+ }++ if (op->src[1]->type != GGML_TYPE_F16) {+ return true;+ }++ if (op->src[0]->type == GGML_TYPE_BF16) {+ return false;+ }++ if (src0_f16_has_mv && op->src[0]->type == GGML_TYPE_F16) {+ return true;+ }++ return mm_path;+}+bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op) {const bool has_simdgroup_mm = dev->props.has_simdgroup_mm;const bool has_simdgroup_reduction = dev->props.has_simdgroup_reduction;@@ -1713,9 +1742,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tecase GGML_OP_GATED_DELTA_NET:return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0;case GGML_OP_SOLVE_TRI:+ return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32;case GGML_OP_MUL_MAT:+ return ggml_metal_supports_mul_mat_op(+ has_simdgroup_reduction, op, true,+ ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm));case GGML_OP_MUL_MAT_ID:- return has_simdgroup_reduction && op->src[0]->type != GGML_TYPE_NVFP4;+ return ggml_metal_supports_mul_mat_op(+ has_simdgroup_reduction, op, false,+ ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm));case GGML_OP_SET:case GGML_OP_CPY:case GGML_OP_DUP:diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex 89c8483b37..7671d1d015 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -2362,10 +2362,6 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {const int16_t r2 = ne12/ne02;const int16_t r3 = ne13/ne03;- // find the break-even point where the matrix-matrix kernel becomes more efficient compared- // to the matrix-vector kernel- const int ne11_mm_min = 8;-// first try to use small-batch mat-mv kernels// these should be efficient for BS [2, ~8]if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) &&@@ -2468,12 +2464,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + r0ptg - 1)/r0ptg), ((ne11 + r1ptg - 1)/r1ptg), ne12*ne13, 32, nsg, 1);- } else if (- !ggml_is_transposed(op->src[0]) &&- !ggml_is_transposed(op->src[1]) &&- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel- props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) {+ } else if (ggml_metal_op_mul_mat_use_mm(op, props_dev->has_simdgroup_mm)) {//GGML_LOG_INFO("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);// some Metal matrix data types require aligned pointers@@ -2622,13 +2613,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {const uint32_t r2 = 1;const uint32_t r3 = 1;- // find the break-even point where the matrix-matrix kernel becomes more efficient compared- // to the matrix-vector kernel- // ne20 = n_used_experts- // ne21 = n_rows (batch size)- const int ne21_mm_id_min = 32;-- if (props_dev->has_simdgroup_mm && ne00 >= 64 && (ne21 >= ne21_mm_id_min)) {+ if (ggml_metal_op_mul_mat_id_use_mm(op, props_dev->has_simdgroup_mm)) {// some Metal matrix data types require aligned pointers// ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)//switch (op->src[0]->type) {