LLM inference in C/C++
C++ ggml
ggml : fix broken CPU concat implementation for quantized types (#25247)
fairydreamingef2d770117db
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex 6724686b8a..c555831ce7 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -1913,7 +1913,11 @@ static void ggml_compute_forward_concat_any(GGML_ASSERT(dim >= 0 && dim < 4);int64_t o[4] = {0, 0, 0, 0};- o[dim] = src0->ne[dim];+ if (dim == 0) {+ o[dim] = src0->ne[dim]/ggml_blck_size(src0->type);+ } else {+ o[dim] = src0->ne[dim];+ }const char * x;@@ -1921,8 +1925,8 @@ static void ggml_compute_forward_concat_any(for (int i3 = 0; i3 < ne3; i3++) {for (int i2 = ith; i2 < ne2; i2 += nth) {for (int i1 = 0; i1 < ne1; i1++) {- for (int i0 = 0; i0 < ne0; i0++) {- if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {+ for (int i0 = 0; i0 < ne0/ggml_blck_size(dst->type); i0++) {+ if (i0 < ne00/ggml_blck_size(src0->type) && i1 < ne01 && i2 < ne02 && i3 < ne03) {x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03;} else {x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13;@@ -2071,6 +2075,14 @@ void ggml_compute_forward_concat(ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];++ if (ggml_is_quantized(src0->type)) {+ GGML_ASSERT(ggml_is_contiguous(src0));+ GGML_ASSERT(ggml_is_contiguous(src1));+ GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0);+ GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0);+ }switch (src0->type) {case GGML_TYPE_F16:diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 5d1f7d8ad5..c21675ef54 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -8918,6 +8918,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}}+ for (ggml_type type_a : { GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }) {+ for (int dim : { 0, 1, 2, 3, }) {+ test_cases.emplace_back(new test_concat(type_a, {128, 12, 13, 14}, dim == 0 ? 256 : 7, dim, 0));+ }+ }+for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) {for (uint32_t i = 4; i <= 1024*1024; i *= 2) {test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1}));