LLM inference in C/C++
C++ ggml
metal : add F16 support for bin ops (#26465)
Georgi Gerganov9d21b57f2e8e
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 6bc60a0db7..20a2fdcfa4 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -1181,6 +1181,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tecase GGML_OP_MUL:case GGML_OP_DIV:case GGML_OP_ADD_ID:+ return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[0]->type == op->src[1]->type);case GGML_OP_ACC:return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32;case GGML_OP_REPEAT:diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex 49f4ee37bb..caa162a5c9 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -3334,9 +3334,6 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {GGML_TENSOR_LOCALS( int32_t, ne, op, ne);GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);- GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);- GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);-GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));GGML_ASSERT(ggml_is_contiguous_rows(op->src[1]));diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metalindex 3c8c0ecd41..6044241193 100644--- a/ggml/src/ggml-metal/ggml-metal.metal+++ b/ggml/src/ggml-metal/ggml-metal.metal@@ -1418,6 +1418,8 @@ typedef decltype(kernel_bin_fuse_impl<float, float, float>) kernel_bin_fuse_t;template [[host_name("kernel_bin_fuse_f32_f32_f32")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<float, float, float>;template [[host_name("kernel_bin_fuse_f32_f32_f32_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<float4, float4, float4>;+template [[host_name("kernel_bin_fuse_f16_f16_f16")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<half, half, half>;+template [[host_name("kernel_bin_fuse_f16_f16_f16_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<half4, half4, half4>;kernel void kernel_add_id(constant ggml_metal_kargs_add_id & args,