LLM inference in C/C++
C++ ggml
vulkan: fix wrong results when a mul_mat reads a slice of a larger cache (#28956)
Pascal52624716156a
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 414e14f856..1521e508c9 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -6001,6 +6001,11 @@ bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) {(tensor->ne[3] == 1 || tensor->nb[3] == tensor->nb[2]*tensor->ne[2]);}+// Batch stride in elements of a tensor read in place.+static uint32_t ggml_vk_batch_stride(const ggml_tensor * tensor) {+ return (uint32_t)(tensor->nb[2] / ggml_type_size(tensor->type) * ggml_blck_size(tensor->type));+}+vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to) {// Choose "contiguous copy" shader if src/dst are contiguous@@ -6488,8 +6493,13 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub}}- uint32_t stride_batch_x = ne00*ne01;- uint32_t stride_batch_y = ne10*ne11;+ // The partial tiles of a quantized A rely on the bound range to read zeros past the last row,+ // so the range stays exact: the strided extent of a tensor read in place, the staged size otherwise.+ const uint64_t x_range = qx_needs_dequant ? x_sz : ggml_nbytes(src0);+ const uint64_t y_range = (qy_needs_dequant || quantize_y) ? y_sz : ggml_nbytes(src1);++ uint32_t stride_batch_x = qx_needs_dequant ? ne00*ne01 : ggml_vk_batch_stride(src0);+ uint32_t stride_batch_y = (qy_needs_dequant || quantize_y) ? ne10*ne11 : ggml_vk_batch_stride(src1);if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) {stride_batch_x = src0->nb[0] / ggml_type_size(src0->type);@@ -6502,7 +6512,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub// computeggml_vk_matmul(ctx, subctx, pipeline,- { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz },+ { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range },ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, d_sz * split_k },ne01, ne11, ne10,ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d,@@ -6770,8 +6780,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&}// For batch_n, the A matrix is the same for each batch, and B/D use the row stride as the batch stride- uint32_t stride_batch_x = batch_n ? 0 : ne00*ne01;- uint32_t stride_batch_y = batch_n ? ne10 : (ne10*ne11);+ uint32_t stride_batch_x = batch_n ? 0 : (qx_needs_dequant ? ne00*ne01 : ggml_vk_batch_stride(src0));+ uint32_t stride_batch_y = batch_n ? ne10 : ((qy_needs_dequant || quantize_y) ? ne10*ne11 : ggml_vk_batch_stride(src1));uint32_t stride_batch_d = batch_n ? ne20 : (ne20*ne21);if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) {@@ -7601,9 +7611,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&}ggml_vk_sync_buffers(ctx, subctx);- uint32_t stride_batch_x = ne00*ne01;+ const uint64_t x_range = qx_needs_dequant ? x_sz : ggml_nbytes(src0);+ const uint64_t y_range = (qy_needs_dequant || quantize_y) ? y_sz : ggml_nbytes(src1);++ uint32_t stride_batch_x = qx_needs_dequant ? ne00*ne01 : ggml_vk_batch_stride(src0);uint32_t stride_b_y = y_needs_k_padding ? y_staged_row_stride : ne10;- uint32_t stride_batch_y = y_needs_k_padding ? y_staged_row_stride * ne11 : ne10*ne11;+ uint32_t stride_batch_y = y_needs_k_padding ? y_staged_row_stride * ne11 : ((qy_needs_dequant || quantize_y) ? ne10*ne11 : ggml_vk_batch_stride(src1));if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) {stride_batch_x = src0->nb[0] / ggml_type_size(src0->type);@@ -7616,7 +7629,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&// computeggml_vk_matmul_id(ctx, subctx, pipeline,- { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz },+ { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range },{ d_D, d_buf_offset, d_sz }, { d_ids, ids_buf_offset, ids_sz }, expert_count_buf,ne01, ne21, ne10, ne10, stride_b_y, ne01,stride_batch_x, stride_batch_y, ne20*ne21,@@ -7801,7 +7814,8 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte}}- uint32_t stride_batch_y = ne10*ne11;+ uint32_t stride_batch_x = qx_needs_dequant ? ne00*ne01 : ggml_vk_batch_stride(src0);+ uint32_t stride_batch_y = (qy_needs_dequant || quantize_y) ? ne10*ne11 : ggml_vk_batch_stride(src1);if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant) {stride_batch_y = src1->nb[2] / ggml_type_size(src1->type);@@ -7844,7 +7858,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_contefor (uint32_t expert_i1 = 0; expert_i1 < nei1; ++expert_i1) {const vk_mat_vec_id_push_constants pc = {(uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01,- (uint32_t)(ne00 * ne01), stride_batch_y, (uint32_t)(ne20 * ne21),+ stride_batch_x, stride_batch_y, (uint32_t)(ne20 * ne21),fusion_flags,(uint32_t)nei0, (uint32_t)ne11, expert_i1, nbi1};diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 32c3f3d2a3..510a294268 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -4954,9 +4954,11 @@ struct test_mul_mat : public test_case {const int64_t k_v; // size of k in memory, resulting in a non-contiguous view for k_v > k, no view for k_v == 0const uint32_t o; // number of outputsconst bool src_overlap; // a and b are overlapping views of the same tensor+ const int64_t m_v; // rows of a in memory, the batches of a are strided for m_v > m, no view for m_v == 0+ const int64_t pad; // bytes after the m_v rows of each batch of a, so nb[2] of a is not a multiple of nb[1]std::string vars() override {- return VARS_TO_STR11(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap);+ return VARS_TO_STR13(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad);}double max_nmse_err() override {@@ -4987,8 +4989,8 @@ struct test_mul_mat : public test_case {std::array<int64_t, 2> bs = {10, 10},std::array<int64_t, 2> nr = {2, 2},std::array<int64_t, 4> per = {0, 1, 2, 3},- int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false)- : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap) {}+ int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0)+ : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad) {}ggml_tensor * build_graph(ggml_context * ctx) override {// C^T = A * B^T: (k, m) * (k, n) => (m, n)@@ -4999,6 +5001,7 @@ struct test_mul_mat : public test_case {if (npermuted > 0) {GGML_ASSERT(npermuted == 2);GGML_ASSERT(k_v == 0); // not handled+ GGML_ASSERT(m_v == 0); // not handledGGML_ASSERT(!ggml_is_quantized(type_a) || per[0] == 0);GGML_ASSERT(!ggml_is_quantized(type_b) || per[0] == 0);@@ -5024,6 +5027,7 @@ struct test_mul_mat : public test_case {} else if (src_overlap) {GGML_ASSERT(type_a == type_b);GGML_ASSERT(k_v == 0);+ GGML_ASSERT(m_v == 0);// a and b are interleaved views of the same tensor: (e.g. fused QKV in MiniMax-01)ggml_tensor * base = ggml_new_tensor_4d(ctx, type_a, 2*k, std::max(m, n), bs[0]*nr[0], bs[1]*nr[1]);@@ -5035,8 +5039,9 @@ struct test_mul_mat : public test_case {ggml_set_name(b, "b");} else {const int64_t k_physical = k_v == 0 ? k : k_v;- a = ggml_new_tensor_4d(ctx, type_a, k_physical, m, bs[0], bs[1]);- b = ggml_new_tensor_4d(ctx, type_b, k_physical, n, bs[0]*nr[0], bs[1]*nr[1]);+ const int64_t m_physical = m_v == 0 ? m : m_v + (pad != 0);+ a = ggml_new_tensor_4d(ctx, type_a, k_physical, m_physical, bs[0], bs[1]);+ b = ggml_new_tensor_4d(ctx, type_b, k_physical, n, bs[0]*nr[0], bs[1]*nr[1]);if (!ggml_is_quantized(type_a)) {if (bs[1] == 1 && nr[1] == 1) {@@ -5050,6 +5055,11 @@ struct test_mul_mat : public test_case {a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], 0);b = ggml_view_4d(ctx, b, k, n, bs[0]*nr[0], bs[1]*nr[1], b->nb[1], b->nb[2], b->nb[3], 0);}+ if (m_v != 0) {+ GGML_ASSERT(m_v > m);+ GGML_ASSERT(pad < (int64_t) a->nb[1]);+ a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], m_v*a->nb[1] + pad, a->nb[3], 0);+ }ggml_set_name(a, "a");ggml_set_name(b, "b");}@@ -5203,9 +5213,10 @@ struct test_mul_mat_id : public test_case {const int64_t n;const int64_t k;const float amax; // magnitude of src1+ const int64_t m_v; // rows of as in memory, the experts of as are strided for m_v > m, no view for m_v == 0std::string vars() override {- return VARS_TO_STR9(type_a, type_b, n_mats, n_used, b, m, n, k, amax);+ return VARS_TO_STR10(type_a, type_b, n_mats, n_used, b, m, n, k, amax, m_v);}double max_nmse_err() override {@@ -5230,15 +5241,19 @@ struct test_mul_mat_id : public test_case {test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,int n_mats = 8, int n_used = 2, bool b = false,int64_t m = 32, int64_t n = 32, int64_t k = 32,- float amax = 1.0f)+ float amax = 1.0f, int64_t m_v = 0): type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b),- m(m), n(n), k(k), amax(amax) {+ m(m), n(n), k(k), amax(amax), m_v(m_v) {GGML_ASSERT(n_used <= n_mats);+ GGML_ASSERT(m_v == 0 || m_v > m);}ggml_tensor * build_graph(ggml_context * ctx) override {// C^T = A * B^T: (k, m) * (k, n) => (m, n)- ggml_tensor * as = ggml_new_tensor_3d(ctx, type_a, k, m, n_mats);+ ggml_tensor * as = ggml_new_tensor_3d(ctx, type_a, k, m_v == 0 ? m : m_v, n_mats);+ if (m_v != 0) {+ as = ggml_view_3d(ctx, as, k, m, n_mats, as->nb[1], as->nb[2], 0);+ }ggml_set_name(as, "as");ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n);@@ -10261,6 +10276,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 16, 32, 32, { 1, 1}, {1, 1}, {0, 1, 2, 3}, 64, 3));test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 77, 77, {12,1}, {1,1}));test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 32, 4, 96, {3, 2}, {1, 1}, {0, 1, 2, 3}, 0, 1, true));+ // the first rows of a KV cache: a is a view whose batches are strided by the cache length+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 8, 1, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 5120));+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 8, 16, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 5120));+ // the batches of a are padded, nb[2] is not a multiple of nb[1]+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 8, 1, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16));+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 8, 16, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16));+ // as is a view whose experts are strided by more rows than it uses+ test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 8, 1, 64, 1.0f, 64));+ test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 4, 2, false, 8, 16, 64, 1.0f, 64));test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1}));test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1, 1}, {1, 1}));