LLM inference in C/C++
C++ ggml
vulkan: support all backend tests for SQR/SQRT/SIN/COS/CLAMP/LEAKY_RELU/NORM (#24582)
Jeff Bolz72a926917229
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex 74611dce7f..6724686b8a 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -3688,8 +3688,6 @@ static void ggml_compute_forward_norm_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));- GGML_ASSERT(src0->nb[0] == sizeof(float));-const int ith = params->ith;const int nth = params->nth;@@ -3703,25 +3701,49 @@ static void ggml_compute_forward_norm_f32(for (int64_t i03 = 0; i03 < ne03; i03++) {for (int64_t i02 = 0; i02 < ne02; i02++) {for (int64_t i01 = ith; i01 < ne01; i01 += nth) {- const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);+ const char * x = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;+ char * y = (char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3;- float sum = 0.0;- ggml_vec_sum_f32(ne00, &sum, x);- float mean = sum/ne00;+ if (nb00 == sizeof(float) && nb0 == sizeof(float)) {+ const float * xf = (const float *) x;- float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);- float variance = 0;+ float sum = 0.0;+ ggml_vec_sum_f32(ne00, &sum, xf);+ float mean = sum/ne00;++ float * yf = (float *) y;+ float variance = 0;#ifdef GGML_USE_ACCELERATE- mean = -mean;- vDSP_vsadd(x, 1, &mean, y, 1, ne00);- vDSP_measqv(y, 1, &variance, ne00);+ mean = -mean;+ vDSP_vsadd(xf, 1, &mean, yf, 1, ne00);+ vDSP_measqv(yf, 1, &variance, ne00);#else- variance = ggml_vec_cvar_f32(ne00, y, x, mean);+ variance = ggml_vec_cvar_f32(ne00, yf, xf, mean);#endif //GGML_USE_ACCELERATE- const float scale = 1.0f/sqrtf(variance + eps);- ggml_vec_scale_f32(ne00, y, scale);+ const float scale = 1.0f/sqrtf(variance + eps);+ ggml_vec_scale_f32(ne00, yf, scale);+ } else {+ float sum = 0.0;+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ sum += *(const float *) (x + i00*nb00);+ }+ const float mean = sum/ne00;++ float variance = 0.0f;+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const float v = *(const float *) (x + i00*nb00) - mean;+ *(float *) (y + i00*nb0) = v;+ variance += v * v;+ }+ variance /= ne00;++ const float scale = 1.0f/sqrtf(variance + eps);+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ *(float *) (y + i00*nb0) *= scale;+ }+ }}}}@@ -4142,8 +4164,6 @@ static void ggml_compute_forward_l2_norm_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));- GGML_ASSERT(src0->nb[0] == sizeof(float));-const int ith = params->ith;const int nth = params->nth;@@ -4158,20 +4178,27 @@ static void ggml_compute_forward_l2_norm_f32(for (int64_t i03 = 0; i03 < ne03; i03++) {for (int64_t i02 = 0; i02 < ne02; i02++) {for (int64_t i01 = ith; i01 < ne01; i01 += nth) {- const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);+ const char * x = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;ggml_float sum = 0.0;for (int64_t i00 = 0; i00 < ne00; i00++) {- sum += (ggml_float)(x[i00] * x[i00]);+ const float xi = *(const float *) (x + i00*nb00);+ sum += (ggml_float)(xi * xi);}- float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);-- memcpy(y, x, ne00 * sizeof(float));-const float scale = 1.0f/fmaxf(sqrtf(sum), eps);- ggml_vec_scale_f32(ne00, y, scale);+ char * y = (char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3;++ if (nb00 == sizeof(float) && nb0 == sizeof(float)) {+ memcpy(y, x, ne00 * sizeof(float));+ ggml_vec_scale_f32(ne00, (float *) y, scale);+ } else {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const float xi = *(const float *) (x + i00*nb00);+ *(float *) (y + i00*nb0) = xi * scale;+ }+ }}}}diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 3d4b5f6056..cca70592f8 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5334,7 +5334,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_OP_NORM:case GGML_OP_RMS_NORM:case GGML_OP_L2_NORM:- return true;+ return ggml_is_contiguous_rows(op->src[0]);case GGML_OP_RMS_NORM_BACK:return ggml_is_contiguous(op->src[0]);break;diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex d2827ad71f..f4a578b893 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -816,14 +816,10 @@ struct vk_device_struct {vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;vk_pipeline pipeline_scale_f32;- vk_pipeline pipeline_sqr_f32;- vk_pipeline pipeline_sqrt_f32;- vk_pipeline pipeline_sin_f32;- vk_pipeline pipeline_cos_f32;vk_pipeline pipeline_log[2];vk_pipeline pipeline_tri[2];vk_pipeline pipeline_diag[2];- vk_pipeline pipeline_clamp_f32;+ vk_pipeline pipeline_clamp[2];vk_pipeline pipeline_pad_f32;vk_pipeline pipeline_roll_f32;vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;@@ -855,6 +851,10 @@ struct vk_device_struct {vk_pipeline pipeline_gelu_quick[2];vk_pipeline pipeline_silu[2];vk_pipeline pipeline_relu[2];+ vk_pipeline pipeline_sqr[2];+ vk_pipeline pipeline_sqrt[2];+ vk_pipeline pipeline_sin[2];+ vk_pipeline pipeline_cos[2];vk_pipeline pipeline_xielu[2];vk_pipeline pipeline_neg[2];vk_pipeline pipeline_tanh[2];@@ -886,7 +886,7 @@ struct vk_device_struct {vk_pipeline pipeline_geglu_erf[2];vk_pipeline pipeline_geglu_quick[2];- vk_pipeline pipeline_leaky_relu_f32;+ vk_pipeline pipeline_leaky_relu[2];vk_pipeline pipeline_silu_back_f32;vk_pipeline pipeline_diag_mask_inf_f32;vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;@@ -4972,7 +4972,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {}ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);@@ -5092,11 +5092,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_sqr_f32, "sqr_f32", sqr_f32_len, sqr_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_sqrt_f32, "sqrt_f32", sqrt_f32_len, sqrt_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_sin_f32, "sin_f32", sin_f32_len, sin_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_cos_f32, "cos_f32", cos_f32_len, cos_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);-ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);@@ -5106,8 +5101,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_diag[0], "diag_f32", diag_f32_len, diag_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_clamp_f32, "clamp_f32", clamp_f32_len, clamp_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);-ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_roll_f32, "roll_f32", roll_f32_len, roll_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);@@ -5127,6 +5120,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {CREATE_UNARY(gelu_quick)CREATE_UNARY(silu)CREATE_UNARY(relu)+ CREATE_UNARY(sqr)+ CREATE_UNARY(sqrt)+ CREATE_UNARY(sin)+ CREATE_UNARY(cos)+ CREATE_UNARY(clamp)+ CREATE_UNARY(leaky_relu)CREATE_UNARY(xielu)CREATE_UNARY(neg)CREATE_UNARY(tanh)@@ -5166,7 +5165,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {CREATE_GLU(geglu_quick)#undef CREATE_GLU- ggml_vk_create_pipeline(device, device->pipeline_leaky_relu_f32, "leaky_relu_f32", leaky_relu_f32_len, leaky_relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {1, 512, 1}, {}, 1, true);@@ -10521,23 +10519,27 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const}return nullptr;case GGML_OP_SQR:- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- return ctx->device->pipeline_sqr_f32;+ if (src0->type == dst->type &&+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {+ return ctx->device->pipeline_sqr[dst->type == GGML_TYPE_F16];}return nullptr;case GGML_OP_SQRT:- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- return ctx->device->pipeline_sqrt_f32;+ if (src0->type == dst->type &&+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {+ return ctx->device->pipeline_sqrt[dst->type == GGML_TYPE_F16];}return nullptr;case GGML_OP_SIN:- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- return ctx->device->pipeline_sin_f32;+ if (src0->type == dst->type &&+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {+ return ctx->device->pipeline_sin[dst->type == GGML_TYPE_F16];}return nullptr;case GGML_OP_COS:- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- return ctx->device->pipeline_cos_f32;+ if (src0->type == dst->type &&+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {+ return ctx->device->pipeline_cos[dst->type == GGML_TYPE_F16];}return nullptr;case GGML_OP_LOG:@@ -10559,8 +10561,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const}return nullptr;case GGML_OP_CLAMP:- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- return ctx->device->pipeline_clamp_f32;+ if (src0->type == dst->type &&+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {+ return ctx->device->pipeline_clamp[dst->type == GGML_TYPE_F16];}return nullptr;case GGML_OP_PAD:@@ -10928,8 +10931,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const}return nullptr;case GGML_OP_LEAKY_RELU:- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- return ctx->device->pipeline_leaky_relu_f32;+ if (src0->type == dst->type &&+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {+ return ctx->device->pipeline_leaky_relu[dst->type == GGML_TYPE_F16];}return nullptr;case GGML_OP_CONV_2D:@@ -11431,6 +11435,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, cocase GGML_OP_TRI:case GGML_OP_DIAG:case GGML_OP_CLAMP:+ case GGML_OP_LEAKY_RELU:case GGML_OP_PAD:case GGML_OP_ROLL:case GGML_OP_REPEAT:@@ -12297,8 +12302,10 @@ static void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx,static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {float * op_params = (float *)dst->op_params;+ vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);+ p.param1 = op_params[0];- ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f });+ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, std::move(p));}static void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {@@ -13399,7 +13406,10 @@ static void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctxstatic void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const float * op_params = (const float *)dst->op_params;- ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, { (uint32_t)ggml_nelements(src0), 0, op_params[0], 0.0f, 0.0f, 0.0f });+ vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);+ p.param1 = op_params[0];++ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p));}#ifdef GGML_VULKAN_RUN_TESTS@@ -17325,12 +17335,11 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmcase GGML_OP_TRANSPOSE:case GGML_OP_RMS_NORM:return true;- case GGML_OP_NORM:case GGML_OP_GROUP_NORM:return ggml_is_contiguous(op->src[0]);+ case GGML_OP_NORM:case GGML_OP_L2_NORM:- return ggml_is_contiguous_rows(op->src[0]) &&- op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;+ return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;case GGML_OP_ADD:case GGML_OP_SUB:case GGML_OP_MUL:@@ -17349,8 +17358,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmcase GGML_OP_SIN:case GGML_OP_COS:case GGML_OP_CLAMP:- return op->src[0]->type == GGML_TYPE_F32;case GGML_OP_LEAKY_RELU:+ return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&+ op->type == op->src[0]->type;case GGML_OP_OPT_STEP_ADAMW:case GGML_OP_OPT_STEP_SGD:return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp b/ggml/src/ggml-vulkan/vulkan-shaders/clamp.compdeleted file mode 100644index 653431895e..0000000000--- a/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp+++ /dev/null@@ -1,17 +0,0 @@-#version 450--#include "types.glsl"-#include "generic_unary_head.glsl"--layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;--void main() {- const uint idx = get_idx();-- if (idx >= p.ne) {- return;- }-- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val < p.param1 ? p.param1 : (val > p.param2 ? p.param2 : val));-}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp b/ggml/src/ggml-vulkan/vulkan-shaders/cos.compdeleted file mode 100644index db6865db98..0000000000--- a/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp+++ /dev/null@@ -1,17 +0,0 @@-#version 450--#include "types.glsl"-#include "generic_unary_head.glsl"--layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;--void main() {- const uint idx = get_idx();-- if (idx >= p.ne) {- return;- }-- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(cos(val));-}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.compindex f9af46744d..9039ed1ded 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp@@ -14,16 +14,13 @@ void main() {const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;const uint tid = gl_LocalInvocationID.x;- const uint i3 = row / (p.ne11 * p.ne12);- const uint i3_offset = i3 * p.ne12 * p.ne11;- const uint i2 = (row - i3_offset) / p.ne11;- const uint i2_offset = i2 * p.ne11;- const uint i1 = row - i3_offset - i2_offset;+ const uint a_base = get_aoffset() + src0_idx(row * p.ne00);+ const uint d_base = get_doffset() + dst_idx(row * p.ne10);sum[tid] = FLOAT_TYPE(0.0f); // partial sum for thread in warp[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {- const FLOAT_TYPE xi = FLOAT_TYPE(data_a[i3*p.nb03 + i2*p.nb02 + i1*p.nb01 + i0]);+ const FLOAT_TYPE xi = FLOAT_TYPE(data_a[a_base + i0*p.nb00]);sum[tid] += xi * xi;}@@ -39,6 +36,6 @@ void main() {const FLOAT_TYPE scale = 1.0f / max(sqrt(sum[0]), FLOAT_TYPE(p.param1));[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {- data_d[i3*p.nb13 + i2*p.nb12 + i1*p.nb11 + i0] = D_TYPE(scale * FLOAT_TYPE(data_a[i3*p.nb03 + i2*p.nb02 + i1*p.nb01 + i0]));+ data_d[d_base + i0*p.nb10] = D_TYPE(scale * FLOAT_TYPE(data_a[a_base + i0*p.nb00]));}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp b/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.compdeleted file mode 100644index b281e855cb..0000000000--- a/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp+++ /dev/null@@ -1,22 +0,0 @@-#version 450--#include "generic_head.glsl"-#include "types.glsl"--#extension GL_EXT_control_flow_attributes : enable--layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;--layout (binding = 0) readonly buffer X {A_TYPE data_a[];};-layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};--void main() {- const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;-- if (i >= p.KX) {- return;- }-- const float val = float(data_a[i]);- data_d[i] = D_TYPE(max(val, 0.0f) + min(val, 0.0f) * p.param1);-}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/norm.compindex cc3ea0b760..792012d57e 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp@@ -1,26 +1,26 @@#version 450-#include "generic_head.glsl"#include "types.glsl"+#include "generic_unary_head.glsl"#extension GL_EXT_control_flow_attributes : enable#define BLOCK_SIZE 512layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;-layout (binding = 0) readonly buffer X {A_TYPE data_a[];};-layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};-shared vec2 sum[BLOCK_SIZE];void main() {const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;const uint tid = gl_LocalInvocationID.x;+ const uint a_base = get_aoffset() + src0_idx(row * p.ne00);+ const uint d_base = get_doffset() + dst_idx(row * p.ne10);+sum[tid] = vec2(0.0f, 0.0f);- [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {- const float xi = float(data_a[row*p.KX + col]);+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {+ const float xi = float(data_a[a_base + i0*p.nb00]);sum[tid].x += xi;sum[tid].y += xi * xi;}@@ -34,11 +34,11 @@ void main() {barrier();}- const float mean = sum[0].x / p.KX;- const float var = sum[0].y / p.KX - mean * mean;+ const float mean = sum[0].x / p.ne00;+ const float var = sum[0].y / p.ne00 - mean * mean;const float inv_std = inversesqrt(var + p.param1);- [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {- data_d[row*p.KX + col] = D_TYPE((float(data_a[row*p.KX + col]) - mean) * inv_std);+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {+ data_d[d_base + i0*p.nb10] = D_TYPE((float(data_a[a_base + i0*p.nb00]) - mean) * inv_std);}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp b/ggml/src/ggml-vulkan/vulkan-shaders/sin.compdeleted file mode 100644index 61f17b2f00..0000000000--- a/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp+++ /dev/null@@ -1,17 +0,0 @@-#version 450--#include "types.glsl"-#include "generic_unary_head.glsl"--layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;--void main() {- const uint idx = get_idx();-- if (idx >= p.ne) {- return;- }-- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(sin(val));-}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sqrt.comp b/ggml/src/ggml-vulkan/vulkan-shaders/sqrt.compdeleted file mode 100644index 70daad6c5d..0000000000--- a/ggml/src/ggml-vulkan/vulkan-shaders/sqrt.comp+++ /dev/null@@ -1,17 +0,0 @@-#version 450--#include "types.glsl"-#include "generic_unary_head.glsl"--layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;--void main() {- const uint idx = get_idx();-- if (idx >= p.ne) {- return;- }-- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(sqrt(val));-}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/square.comp b/ggml/src/ggml-vulkan/vulkan-shaders/square.compdeleted file mode 100644index 4eb56afcb1..0000000000--- a/ggml/src/ggml-vulkan/vulkan-shaders/square.comp+++ /dev/null@@ -1,17 +0,0 @@-#version 450--#include "types.glsl"-#include "generic_unary_head.glsl"--layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;--void main() {- const uint idx = get_idx();-- if (idx >= p.ne) {- return;- }-- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val * val);-}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.compindex 47a4573996..c62bce8255 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp@@ -17,6 +17,30 @@ float op_neg(float x) {return -x;}+float op_sqr(float x) {+ return x * x;+}++float op_sqrt(float x) {+ return sqrt(x);+}++float op_sin(float x) {+ return sin(x);+}++float op_cos(float x) {+ return cos(x);+}++float op_clamp(float x) {+ return clamp(x, p.param1, p.param2);+}++float op_leaky_relu(float x) {+ return max(x, 0.0f) + min(x, 0.0f) * p.param1;+}+float op_step(float x) {return x >= 0.0f ? 1.0f : 0.0f;}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cppindex 502602f799..3bd93d256c 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp@@ -849,16 +849,6 @@ void process_shaders() {string_to_spv("scale_f32", "scale.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});- string_to_spv("sqr_f32", "square.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});-- string_to_spv("sqrt_f32", "sqrt.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});-- string_to_spv("sin_f32", "sin.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});-- string_to_spv("cos_f32", "cos.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});-- string_to_spv("clamp_f32", "clamp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});-string_to_spv("pad_f32", "pad.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});string_to_spv("concat_i8", "concat.comp", {{"A_TYPE", "uint8_t"}, {"B_TYPE", "uint8_t"}, {"D_TYPE", "uint8_t"}});@@ -885,6 +875,18 @@ void process_shaders() {string_to_spv("silu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_silu"}});string_to_spv("relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_relu"}});string_to_spv("relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_relu"}});+ string_to_spv("sqr_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqr"}});+ string_to_spv("sqr_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqr"}});+ string_to_spv("sqrt_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqrt"}});+ string_to_spv("sqrt_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqrt"}});+ string_to_spv("sin_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sin"}});+ string_to_spv("sin_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sin"}});+ string_to_spv("cos_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_cos"}});+ string_to_spv("cos_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_cos"}});+ string_to_spv("clamp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_clamp"}});+ string_to_spv("clamp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_clamp"}});+ string_to_spv("leaky_relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_leaky_relu"}});+ string_to_spv("leaky_relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_leaky_relu"}});string_to_spv("neg_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_neg"}});string_to_spv("neg_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_neg"}});string_to_spv("tanh_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_tanh"}});@@ -942,7 +944,6 @@ void process_shaders() {string_to_spv("geglu_quick_f16","geglu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});string_to_spv("geglu_quick_f32","geglu_quick.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});- string_to_spv("leaky_relu_f32", "leaky_relu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});string_to_spv("diag_mask_inf_f32", "diag_mask_inf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cppindex e8eafd185a..f0ec18abd9 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp@@ -4270,7 +4270,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, constcase GGML_OP_RMS_NORM:case GGML_OP_NORM:case GGML_OP_L2_NORM:- supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32;+ supports_op = (op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32) && ggml_is_contiguous_rows(src0);break;case GGML_OP_ROPE:supports_op = op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 719ae51cc2..e1d3853e43 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -3298,21 +3298,29 @@ struct test_norm : public test_case {const std::array<int64_t, 4> ne;const bool v; // whether a is a non-contiguous viewconst float eps;+ const bool noncontig_rows;std::string vars() override {- return VARS_TO_STR4(type, ne, v, eps);+ return VARS_TO_STR5(type, ne, v, eps, noncontig_rows);}test_norm(ggml_type type = GGML_TYPE_F32,std::array<int64_t, 4> ne = {64, 5, 4, 3},bool v = false,- float eps = 1e-6f)- : type(type), ne(ne), v(v), eps(eps) {}+ float eps = 1e-6f,+ bool noncontig_rows = false)+ : type(type), ne(ne), v(v), eps(eps), noncontig_rows(noncontig_rows) {}ggml_tensor * build_graph(ggml_context * ctx) override {- ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());+ const std::array<int64_t, 4> ne_a = noncontig_rows ?+ std::array<int64_t, 4>{ ne[1], ne[0], ne[2], ne[3] } : ne;+ ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data());ggml_set_name(a, "a");+ if (noncontig_rows) {+ a = ggml_permute(ctx, a, 1, 0, 2, 3);+ ggml_set_name(a, "permuted a");+ }if (v) {a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0);ggml_set_name(a, "view of a");@@ -6193,21 +6201,29 @@ struct test_l2_norm : public test_case {const std::array<int64_t, 4> ne;const float eps;bool v;+ bool noncontig_rows;std::string vars() override {- return VARS_TO_STR4(type, ne, eps, v);+ return VARS_TO_STR5(type, ne, eps, v, noncontig_rows);}test_l2_norm(ggml_type type = GGML_TYPE_F32,std::array<int64_t, 4> ne = {64, 64, 320, 1},float eps = 1e-12f,- bool v = false)- : type(type), ne(ne), eps(eps), v(v) {}+ bool v = false,+ bool noncontig_rows = false)+ : type(type), ne(ne), eps(eps), v(v), noncontig_rows(noncontig_rows) {}ggml_tensor * build_graph(ggml_context * ctx) override {- ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());+ const std::array<int64_t, 4> ne_a = noncontig_rows ?+ std::array<int64_t, 4>{ ne[1], ne[0], ne[2], ne[3] } : ne;+ ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data());ggml_set_name(a, "a");+ if (noncontig_rows) {+ a = ggml_permute(ctx, a, 1, 0, 2, 3);+ ggml_set_name(a, "permuted a");+ }if (v) {a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0);ggml_set_name(a, "view of a");@@ -8282,9 +8298,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps));test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps));}+ test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, false, eps, true));test_cases.emplace_back(new test_rms_norm_back(GGML_TYPE_F32, { n, 5, 4, 3 }, eps));test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false));test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true));+ test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, true));}}