LLM inference in C/C++
C++ ggml
ggml: support non-contiguous SUM (CPU, CUDA, Vulkan, WebGPU) (#28851)
Andrea Rossetti0878f7c304cc
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex 7a73e138ef..7dff1d11b1 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -1292,24 +1292,37 @@ static void ggml_compute_forward_sum_f32(}assert(ggml_is_scalar(dst));- assert(src0->nb[0] == sizeof(float));GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)- ggml_float sum = 0;- ggml_float row_sum = 0;+ ggml_float sum = 0;- for (int64_t i03 = 0; i03 < ne03; i03++) {- for (int64_t i02 = 0; i02 < ne02; i02++) {- for (int64_t i01 = 0; i01 < ne01; i01++) {- ggml_vec_sum_f32_ggf(ne00,- &row_sum,- (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));- sum += row_sum;+ if (nb00 == sizeof(float)) {+ ggml_float row_sum = 0;+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ ggml_vec_sum_f32_ggf(ne00,+ &row_sum,+ (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));+ sum += row_sum;+ }+ }+ }+ } else {+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ const char * row = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ sum += (ggml_float) *(const float *) (row + i00*nb00);+ }+ }}}}+((float *) dst->data)[0] = sum;}diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex cb85321f57..dc30492fa5 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5897,7 +5897,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g//return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]);return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);case GGML_OP_SUM:- return ggml_is_contiguous_rows(op->src[0]);+ return ggml_is_contiguously_allocated(op->src[0]);case GGML_OP_TOP_K:return op->src[0]->ne[0] <= INT_MAX;case GGML_OP_ARGSORT:diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex fa038bb5b6..e603cbcb11 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -15872,6 +15872,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmreturn ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32&& ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32;case GGML_OP_SUM:+ return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguously_allocated(op->src[0]);case GGML_OP_SUM_ROWS:case GGML_OP_MEAN:return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]);diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cppindex 15e6c784b7..77fbb4f476 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp@@ -4781,6 +4781,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, constsupports_op = op->type == GGML_TYPE_F32 && src0->type == op->type;break;case GGML_OP_SUM:+ supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguously_allocated(src0);+ break;case GGML_OP_SUM_ROWS:supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguous_rows(src0);break;diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 011f93865d..8fc90e037b 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -7804,24 +7804,43 @@ struct test_sum : public test_case {const std::array<int64_t, 4> ne;const std::array<int64_t, 4> permute;bool _use_permute;+ const bool slice;std::string vars() override {std::string v = VARS_TO_STR2(type, ne);if (_use_permute) v += "," + VAR_TO_STR(permute);+ if (slice) v += "," + VAR_TO_STR(slice);return v;}test_sum(ggml_type type = GGML_TYPE_F32,std::array<int64_t, 4> ne = {10, 5, 4, 3},- std::array<int64_t, 4> permute = {0, 0, 0, 0})+ std::array<int64_t, 4> permute = {0, 0, 0, 0},+ bool slice = false): type(type), ne(ne), permute(permute),- _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {}+ _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0),+ slice(slice) {}ggml_tensor * build_graph(ggml_context * ctx) override {ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());- ggml_set_param(a);++ // the backward pass of a sliced view scatters the gradient via GGML_OP_ACC,+ // whose CPU implementation requires contiguous rows; a row-breaking permute+ // (permute[0] != 0) on top of a slice violates that, so skip grad there+ const bool grad_supported = !(slice && _use_permute && permute[0] != 0);+ if (grad_supported) {+ ggml_set_param(a);+ }+ggml_set_name(a, "a");+ if (slice) {+ a = ggml_view_4d(ctx, a,+ ne[0], ne[1], ne[2] / 2, ne[3] - 1,+ a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]);+ ggml_set_name(a, "a_sliced");+ }+if (_use_permute) {a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]);ggml_set_name(a, "a_permuted");@@ -11540,6 +11559,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 }));test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }));test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }, { 1, 0, 2, 3 })); // sum dst not-contiguous+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 10, 5, 4, 3 }, { 1, 0, 2, 3 })); // non-contiguous rows, 4D+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 11, 5, 6, 3 }, { 2, 0, 1, 3 })); // rotated dims, nb[0] != sizeof(float)+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 0, 0, 0}, true)); // sliced view (gaps)+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {1, 0, 2, 3}, true)); // sliced + non-contiguous rowstest_cases.emplace_back(new test_sum_rows());test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false));test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true));