LLM inference in C/C++
C++ ggml
vulkan: Support quantized concat (#25684)
Jeff Bolzea63b4d32ea1
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 8a1bff65a5..e32dd7a56a 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -1481,6 +1481,11 @@ struct vk_op_binary_push_constants {float param1; float param2; int32_t param3;};+// Distinct type with the same layout so concat can overload tensor offset initialization.+struct vk_op_concat_push_constants : vk_op_binary_push_constants {};+static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants));+static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>);+struct vk_op_multi_add_push_constants {// shape for dstuint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23;@@ -2246,6 +2251,40 @@ static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggreturn ((vk_tensor_offset(t) + t->view_offs) & (ctx->device->properties.limits.minStorageBufferOffsetAlignment - 1));;}+static uint32_t ggml_vk_concat_unit_size(ggml_type type) {+ const uint32_t type_size = ggml_type_size(type);++ if (!ggml_is_quantized(type)) {+ return type_size;+ }++ // Use the widest existing concat shader that evenly divides a quant block.+ if (type_size % 8 == 0) {+ return 8;+ }+ if (type_size % 4 == 0) {+ return 4;+ }+ if (type_size % 2 == 0) {+ return 2;+ }+ return 1;+}++static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {+ if (src0->type != src1->type || src0->type != dst->type) {+ return false;+ }++ if (!ggml_is_quantized(src0->type)) {+ const size_t type_size = ggml_type_size(src0->type);+ return type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8;+ }++ // Quantized tensor rows are block-aligned when created.+ return ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous_rows(dst);+}+template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {GGML_UNUSED(p);GGML_UNUSED(src0);@@ -10896,14 +10935,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const}return nullptr;case GGML_OP_CONCAT: {- if (src0->type != src1->type || src0->type != dst->type) {- return nullptr;- }- if (ggml_blck_size(src0->type) != 1) {+ if (!ggml_vk_concat_supported(src0, src1, dst)) {return nullptr;}- const size_t type_size = ggml_type_size(src0->type);- switch (type_size) {+ switch (ggml_vk_concat_unit_size(src0->type)) {case 1:return ctx->device->pipeline_concat_i8;case 2:@@ -11595,6 +11630,18 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vkGGML_UNUSED(src3);}+template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size;+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size;+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size;++ p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;++ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}+template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);@@ -11630,7 +11677,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co}std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3];std::cerr << "), " << ggml_op_name(op) << ")");- GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT+ GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || op == GGML_OP_CONCAT || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINTGGML_ASSERT(dst->buffer != nullptr);const uint64_t ne00 = src0->ne[0];const uint64_t ne01 = src0->ne[1];@@ -11885,6 +11932,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, cone *= ggml_type_size(src0->type) / 2;}}+ if (op == GGML_OP_CONCAT && ggml_is_quantized(dst->type)) {+ ne = ne / ggml_blck_size(dst->type) * ggml_type_size(dst->type) / ggml_vk_concat_unit_size(dst->type);+ }// copy_to_quant has block size of 32, and each thread does QUANT_K elements.// Splitting into 512x512xZ wouldn't work well since each workgroup does 1024 elements.// So divide by block size here before splitting into 512x512 groups.@@ -12525,18 +12575,28 @@ static void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subcstatic void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {int * op_params = (int *)dst->op_params;- const uint32_t src0_type_size = ggml_type_size(src0->type);- const uint32_t src1_type_size = ggml_type_size(src1->type);- const uint32_t dst_type_size = ggml_type_size(dst->type);-- ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, {- (uint32_t)ggml_nelements(dst),- (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,- (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,- (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size,+ const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);+ const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size;+ const uint32_t block_size = ggml_blck_size(dst->type);+ const bool quantized = ggml_is_quantized(dst->type);++ // Address dimension 0 in packed storage units; higher strides may be noncontiguous.+ const uint32_t ne00 = src0->ne[0] / block_size * units_per_block;+ const uint32_t ne10 = src1->ne[0] / block_size * units_per_block;+ const uint32_t ne20 = dst->ne[0] / block_size * units_per_block;+ const uint32_t nb00 = quantized ? 1 : src0->nb[0] / unit_size;+ const uint32_t nb10 = quantized ? 1 : src1->nb[0] / unit_size;+ const uint32_t nb20 = quantized ? 1 : dst->nb[0] / unit_size;++ vk_op_concat_push_constants pc {{+ ne20 * (uint32_t)dst->ne[1] * (uint32_t)dst->ne[2] * (uint32_t)dst->ne[3],+ ne00, (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], nb00, (uint32_t)src0->nb[1] / unit_size, (uint32_t)src0->nb[2] / unit_size, (uint32_t)src0->nb[3] / unit_size,+ ne10, (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], nb10, (uint32_t)src1->nb[1] / unit_size, (uint32_t)src1->nb[2] / unit_size, (uint32_t)src1->nb[3] / unit_size,+ ne20, (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], nb20, (uint32_t) dst->nb[1] / unit_size, (uint32_t) dst->nb[2] / unit_size, (uint32_t) dst->nb[3] / unit_size,0,0.0f, 0.0f, op_params[0],- });+ }};+ ggml_vk_op_f32<vk_op_concat_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, std::move(pc));}static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {@@ -17872,12 +17932,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmreturn op->src[0]->type == op->src[1]->type && op->src[0]->type == op->type &&(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_I32);case GGML_OP_CONCAT: {- if (op->src[0]->type != op->src[1]->type || op->src[0]->type != op->type) {- return false;- }- const size_t type_size = ggml_type_size(op->type);- return ggml_blck_size(op->type) == 1 &&- (type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8);+ return ggml_vk_concat_supported(op->src[0], op->src[1], op);}case GGML_OP_ADD1:return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32)