LLM inference in C/C++
C++ ggml
vulkan: support type-aligned GET_ROWS (#28253)
Jeff Bolz0cae43063cf1
diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cppindex 4b1789713d..a795622783 100644--- a/ggml/src/ggml-openvino/ggml-openvino.cpp+++ b/ggml/src/ggml-openvino/ggml-openvino.cpp@@ -1091,6 +1091,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {if (op->ne[3] != 1) {return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};}+ if (op->op == GGML_OP_GET_ROWS && ggml_is_quantized(op->src[0]->type) &&+ op->src[0]->view_src != nullptr && op->src[0]->view_offs != 0) {+ return {false, "GET_ROWS with a nonzero quantized src0 view offset is not supported"};+ }if (op->op == GGML_OP_GET_ROWS && ggml_openvino_get_device_name() == "GPU" &&op->src[0]->type == GGML_TYPE_BF16) {return {false, "GET_ROWS with BF16 src0 is not supported on GPU"};diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 9b47c6c958..62f90847b0 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -2515,9 +2515,38 @@ static uint64_t vk_tensor_offset(const ggml_tensor * tensor) {return (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base;}-static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t)-{- return ((vk_tensor_offset(t) + t->view_offs) & (ctx->device->properties.limits.minStorageBufferOffsetAlignment - 1));;+static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset);++static size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {+ // vk_tensor_offset() is relative to vk_ptr_base, but mapped host tensors need an offset relative to their Vulkan buffer.+ if (ctx->device->uma) {+ vk_buffer buf = nullptr;+ size_t off = 0;+ ggml_vk_host_get(ctx->device, t->data, buf, off);+ if (buf) {+ return off;+ }+ }+ return (size_t)(vk_tensor_offset(t) + t->view_offs);+}++static size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size) {+ // Move the descriptor back until its distance to the tensor is divisible by the tensor type size.+ size_t descriptor_offset = tensor_offset & ~(alignment - 1);+ while ((tensor_offset - descriptor_offset) % type_size != 0) {+ GGML_ASSERT(descriptor_offset >= alignment);+ descriptor_offset -= alignment;+ }++ return descriptor_offset;+}++static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {+ const size_t tensor_offset = ggml_vk_tensor_buffer_offset(ctx, t);+ const size_t descriptor_offset = ggml_vk_descriptor_offset(+ tensor_offset, ctx->device->properties.limits.minStorageBufferOffsetAlignment, ggml_type_size(t->type));+ GGML_ASSERT(tensor_offset - descriptor_offset <= UINT32_MAX);+ return tensor_offset - descriptor_offset;}static uint32_t ggml_vk_concat_unit_size(ggml_type type) {@@ -8265,10 +8294,12 @@ static vk_subbuffer ggml_vk_tensor_subbuffer(size_t size = ggml_nbytes(tensor);- size_t misalign_bytes = offset & (ctx->device->properties.limits.minStorageBufferOffsetAlignment - 1);+ const size_t descriptor_offset = ggml_vk_descriptor_offset(+ offset, ctx->device->properties.limits.minStorageBufferOffsetAlignment, ggml_type_size(tensor->type));+ const size_t misalign_bytes = offset - descriptor_offset;// The shader must support misaligned offsets when indexing into the bufferGGML_ASSERT(allow_misalign || misalign_bytes == 0);- offset &= ~misalign_bytes;+ offset = descriptor_offset;size += misalign_bytes;return vk_subbuffer{buffer, offset, size};@@ -12155,7 +12186,9 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vkconst uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);- GGML_ASSERT(dst->op != GGML_OP_GET_ROWS || (a_offset == 0 && b_offset == 0 && d_offset == 0));+ GGML_ASSERT(a_offset <= 0xFFFF);+ GGML_ASSERT(b_offset <= 0xFF);+ GGML_ASSERT(d_offset <= 0xFF);p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp b/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.compindex 9dba437edb..19af30ac98 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp@@ -27,10 +27,10 @@ void main() {const uint i11 = gid_z / p.ne12;const uint i12 = gid_z % p.ne12;- const uint i01 = data_b[i10*p.nb10 + i11*p.nb11 + i12*p.nb12];+ const uint i01 = data_b[get_boffset() + i10*p.nb10 + i11*p.nb11 + i12*p.nb12];- const uint a_offset = i01*p.nb01 + i11*p.nb02 + i12*p.nb03;- const uint d_offset = i10*p.nb21 + i11*p.nb22 + i12*p.nb23;+ const uint a_offset = get_aoffset() + i01*p.nb01 + i11*p.nb02 + i12*p.nb03;+ const uint d_offset = get_doffset() + i10*p.nb21 + i11*p.nb22 + i12*p.nb23;const uint ib = a_offset + i00/QUANT_K; // block indexconst uint iqs = (i00%QUANT_K)/QUANT_R; // quant indexdiff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cppindex 1a43c72733..2e6c5a8c5e 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp@@ -4323,13 +4323,21 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, constop->type == GGML_TYPE_Q4_0) &&src0->type == GGML_TYPE_F32 && (src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32));break;- case GGML_OP_GET_ROWS:+ case GGML_OP_GET_ROWS: {+ const size_t storage_alignment =+ ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment;+ const size_t src_address_unit =+ src0->type == GGML_TYPE_F32 && op->ne[0] % 4 == 0 ? 4 * sizeof(float) : ggml_type_size(src0->type);+ if (ggml_webgpu_tensor_misalignment(src0, storage_alignment) % src_address_unit != 0) {+ break;+ }if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_webgpu_supported_qtype(src0->type)) {supports_op = (op->type == GGML_TYPE_F32);} else if (src0->type == GGML_TYPE_I32) {supports_op = op->type == GGML_TYPE_I32;}break;+ }case GGML_OP_MUL_MAT:{switch (src1->type) {diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 6a8cfd46d0..eeaca940fd 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -2336,27 +2336,40 @@ struct test_get_rows : public test_case {const int r; // rows to getconst int be1; // batch sizeconst int be2; // batch size- const bool v; // view (non-contiguous src1)+ const bool v; // view src1+ const bool vs0; // view src0std::string vars() override {- return VARS_TO_STR7(type, n, m, r, be1, be2, v);+ return VARS_TO_STR8(type, n, m, r, be1, be2, v, vs0);}- test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false)- : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v) {}+ test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false, bool vs0 = false)+ : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v), vs0(vs0) {}ggml_tensor * build_graph(ggml_context * ctx) override {- ggml_tensor * in = ggml_new_tensor_4d(ctx, type, n, m, be1, be2);- ggml_set_name(in, "in");+ ggml_tensor * in;+ if (vs0) {+ const int offset_rows = 3;+ const int padded_m = m + offset_rows;+ ggml_tensor * in_padded = ggml_new_tensor_4d(ctx, type, n, padded_m, be1, be2);+ ggml_set_name(in_padded, "in_padded");+ in = ggml_view_4d(ctx, in_padded, n, m, be1, be2,+ in_padded->nb[1], in_padded->nb[2], in_padded->nb[3],+ offset_rows * in_padded->nb[1]);+ ggml_set_name(in, "in_view");+ } else {+ in = ggml_new_tensor_4d(ctx, type, n, m, be1, be2);+ ggml_set_name(in, "in");+ }- ggml_tensor * rows = ggml_new_tensor_3d(ctx, GGML_TYPE_I32, r, be1, be2);+ ggml_tensor * rows = ggml_new_tensor_3d(ctx, GGML_TYPE_I32, v ? r + 1 : r, be1, be2);ggml_set_name(rows, "rows");if (v) {- rows = ggml_view_3d(ctx, rows, r/2, be1, be2, rows->nb[1], rows->nb[2], 0);+ rows = ggml_view_3d(ctx, rows, r/2, be1, be2, rows->nb[1], rows->nb[2], rows->nb[0]);ggml_set_name(rows, "view_of_rows");}- const bool grad_supported = ggml_is_matrix(in) && ggml_is_vector(rows);+ const bool grad_supported = !vs0 && ggml_is_matrix(in) && ggml_is_vector(rows);if (grad_supported) {ggml_set_param(in);// rows is a constant input -> no gradients@@ -2370,14 +2383,16 @@ struct test_get_rows : public test_case {void initialize_tensors(ggml_context * ctx) override {for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {+ if (ggml_is_view_op(t->op)) {+ continue;+ }if (t->type == GGML_TYPE_I32) {- if (ggml_is_view_op(t->op)) { continue; }// rows- std::vector<int> data(r*be1*be2);- for (int i = 0; i < r*be1*be2; i++) {+ std::vector<int> data(ggml_nelements(t));+ for (size_t i = 0; i < data.size(); i++) {data[i] = rand() % m;}- ggml_backend_tensor_set(t, data.data(), 0, r * be1 * be2 * sizeof(int));+ ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(int));} else {init_tensor_uniform(t);}@@ -8848,13 +8863,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {for (ggml_type type : all_types) {for (int b : {1, 7}) {for (bool v : {false, true}) {- test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, b, 1, v));+ for (bool vs0 : {false, true}) {+ test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, b, 1, v, vs0));+ }}}}for (int b : {1, 7}) {for (bool v : {false, true}) {- test_cases.emplace_back(new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v));+ for (bool vs0 : {false, true}) {+ test_cases.emplace_back(new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v, vs0));+ }}}