LLM inference in C/C++
C++ ggml
metal : add concat support for quantized types (#28116)
Georgi Gerganovab0b3bd3c846
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 83344539e5..0d8484d002 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -1538,6 +1538,12 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tereturn true;case GGML_TYPE_BF16:return has_bfloat;+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ return true;default:return false;}diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex 30ea4ec274..bc8b3c8d48 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -552,8 +552,24 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {const int32_t dim = ((const int32_t *) op->op_params)[0];+ const bool is_q = ggml_is_quantized(op->type);++ // for quantized types, concat is done at the block level (nb0 == type_size == block size)+ int32_t ne00_arg = ne00;+ int32_t ne10_arg = ne10;+ int32_t ne0_arg = ne0;+ if (is_q) {+ const int32_t blck = ggml_blck_size(op->type);+ GGML_ASSERT(ne00 % blck == 0);+ GGML_ASSERT(ne10 % blck == 0);+ GGML_ASSERT(ne0 % blck == 0);+ ne00_arg = ne00/blck;+ ne10_arg = ne10/blck;+ ne0_arg = ne0/blck;+ }+ggml_metal_kargs_concat args = {- /*.ne00 =*/ ne00,+ /*.ne00 =*/ ne00_arg,/*.ne01 =*/ ne01,/*.ne02 =*/ ne02,/*.ne03 =*/ ne03,@@ -561,7 +577,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {/*.nb01 =*/ nb01,/*.nb02 =*/ nb02,/*.nb03 =*/ nb03,- /*.ne10 =*/ ne10,+ /*.ne10 =*/ ne10_arg,/*.ne11 =*/ ne11,/*.ne12 =*/ ne12,/*.ne13 =*/ ne13,@@ -569,7 +585,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {/*.nb11 =*/ nb11,/*.nb12 =*/ nb12,/*.nb13 =*/ nb13,- /*.ne0 =*/ ne0,+ /*.ne0 =*/ ne0_arg,/*.ne1 =*/ ne1,/*.ne2 =*/ ne2,/*.ne3 =*/ ne3,@@ -588,7 +604,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);- int nth = std::min(256, ne0);+ int nth = std::min(256, ne0_arg);// when rows are small, we can batch them together in a single threadgroupint nrptg = 1;diff --git a/ggml/src/ggml-metal/kernels/quantize.metal b/ggml/src/ggml-metal/kernels/quantize.metalindex 59d0afe969..42ca6d74a0 100644--- a/ggml/src/ggml-metal/kernels/quantize.metal+++ b/ggml/src/ggml-metal/kernels/quantize.metal@@ -207,6 +207,51 @@ template [[host_name("kernel_concat_i16")]] kernel kernel_concat_t kernel_concatemplate [[host_name("kernel_concat_i32")]] kernel kernel_concat_t kernel_concat<int>;template [[host_name("kernel_concat_i64")]] kernel kernel_concat_t kernel_concat<long>;+template<typename block_q>+kernel void kernel_concat_q(+ constant ggml_metal_kargs_concat & args,+ device const char * src0,+ device const char * src1,+ device char * dst,+ uint3 tgpig[[threadgroup_position_in_grid]],+ ushort3 tpitg[[thread_position_in_threadgroup]],+ ushort3 ntg[[threads_per_threadgroup]]) {++ // note: for quantized types, the args are in units of blocks (nb0 == type_size)+ const int i3 = tgpig.z;+ const int i2 = tgpig.y;+ const int i1 = ntg.y == 1 ? tgpig.x : tgpig.x*ntg.y + tpitg.y;++ if (i1 >= args.ne1) {+ return;+ }++ int o[4] = {0, 0, 0, 0};+ o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03));++ for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {+ device const block_q * x;++ if (i0 < args.ne00 && i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) {+ x = (device const block_q *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00);+ } else {+ x = (device const block_q *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10);+ }++ device block_q * y = (device block_q *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);++ *y = *x;+ }+}++typedef decltype(kernel_concat_q<block_q4_0>) kernel_concat_q_t;++template [[host_name("kernel_concat_q4_0")]] kernel kernel_concat_q_t kernel_concat_q<block_q4_0>;+template [[host_name("kernel_concat_q4_1")]] kernel kernel_concat_q_t kernel_concat_q<block_q4_1>;+template [[host_name("kernel_concat_q5_0")]] kernel kernel_concat_q_t kernel_concat_q<block_q5_0>;+template [[host_name("kernel_concat_q5_1")]] kernel kernel_concat_q_t kernel_concat_q<block_q5_1>;+template [[host_name("kernel_concat_q8_0")]] kernel kernel_concat_q_t kernel_concat_q<block_q8_0>;+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>kernel void kernel_get_rows_q(constant ggml_metal_kargs_get_rows & args,