LLM inference in C/C++
C++ ggml
ggml: add SWIGLU_CLAMP (#27930)
Aman Gupta0190529ec450
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.hindex 5f6774a630..26f31232f6 100644--- a/ggml/include/ggml.h+++ b/ggml/include/ggml.h@@ -627,6 +627,7 @@ extern "C" {GGML_GLU_OP_SWIGLU_OAI,GGML_GLU_OP_GEGLU_ERF,GGML_GLU_OP_GEGLU_QUICK,+ GGML_GLU_OP_SWIGLU_CLAMP,GGML_GLU_OP_COUNT,};@@ -1367,6 +1368,12 @@ extern "C" {float alpha,float limit);+ GGML_API struct ggml_tensor * ggml_swiglu_clamp(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ float limit);+// normalize along rowsGGML_API struct ggml_tensor * ggml_norm(struct ggml_context * ctx,diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cppindex 2dc0f40917..902d2eda69 100644--- a/ggml/src/ggml-cann/aclnn_ops.cpp+++ b/ggml/src/ggml-cann/aclnn_ops.cpp@@ -211,6 +211,50 @@ void ggml_cann_swiglu(ggml_backend_cann_context & ctx, ggml_tensor * dst) {GGML_CANN_CALL_ACLNN_OP(ctx, SwiGlu, acl_src.get(), (int64_t)2, acl_dst.get());}+void ggml_cann_swiglu_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {+ ggml_tensor * src0 = dst->src[0];+ ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_is_contiguous_1(src0));+ GGML_ASSERT(ggml_is_contiguous_1(dst));++ const int32_t swapped = ggml_get_op_params_i32(dst, 1);+ acl_tensor_ptr acl_gate;+ acl_tensor_ptr acl_up;+ if (src1) {+ GGML_ASSERT(ggml_is_contiguous_1(src1));+ GGML_ASSERT(src0->type == src1->type);+ acl_gate = ggml_cann_create_tensor(src0);+ acl_up = ggml_cann_create_tensor(src1);+ } else {+ int64_t ne[] = { src0->ne[0] / 2, src0->ne[1], src0->ne[2], src0->ne[3] };+ size_t nb[] = { src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3] };+ acl_gate = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, 0);+ acl_up = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, ne[0] * ggml_element_size(src0));+ if (swapped) {+ std::swap(acl_gate, acl_up);+ }+ }++ ggml_cann_pool_alloc temp_alloc(ctx.pool(), ggml_nbytes(dst));+ acl_tensor_ptr acl_temp = ggml_cann_create_tensor(temp_alloc.get(), ggml_cann_type_mapping(dst->type),+ ggml_element_size(dst), dst->ne, dst->nb, GGML_MAX_DIMS);+ acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);++ const float limit = ggml_get_op_params_f32(dst, 3);+ float min_gate = -INFINITY;+ float min_up = -limit;+ float max_value = limit;+ acl_scalar_ptr acl_min_gate = ggml_cann_create_scalar(&min_gate, ACL_FLOAT);+ acl_scalar_ptr acl_min_up = ggml_cann_create_scalar(&min_up, ACL_FLOAT);+ acl_scalar_ptr acl_limit = ggml_cann_create_scalar(&max_value, ACL_FLOAT);++ GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_gate.get(), acl_min_gate.get(), acl_limit.get(), acl_temp.get());+ GGML_CANN_CALL_ACLNN_OP(ctx, Silu, acl_temp.get(), acl_dst.get());+ GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_up.get(), acl_min_up.get(), acl_limit.get(), acl_temp.get());+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_dst.get(), acl_temp.get());+}+// Fused GeGLU using aclnnGeGluV3: splits input along ne[0] (CANN last dim),// activates the LEFT half with GELU, multiplies by right half.// approximate: 0=tanh, 1=none(erf). activateLeft=true matches GGML convention.@@ -4433,4 +4477,3 @@ void ggml_cann_gated_linear_attn(ggml_backend_cann_context & ctx, ggml_tensor *}}}-diff --git a/ggml/src/ggml-cann/aclnn_ops.h b/ggml/src/ggml-cann/aclnn_ops.hindex cdbf9260f8..678f4d654e 100644--- a/ggml/src/ggml-cann/aclnn_ops.h+++ b/ggml/src/ggml-cann/aclnn_ops.h@@ -76,6 +76,7 @@void ggml_cann_repeat(ggml_backend_cann_context & ctx, ggml_tensor * dst);void ggml_cann_swiglu(ggml_backend_cann_context & ctx, ggml_tensor * dst);+void ggml_cann_swiglu_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst);void ggml_cann_geglu(ggml_backend_cann_context & ctx, ggml_tensor * dst, int64_t approximate);/**diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cppindex 5e5541aac9..c2745014a1 100644--- a/ggml/src/ggml-cann/ggml-cann.cpp+++ b/ggml/src/ggml-cann/ggml-cann.cpp@@ -1872,6 +1872,9 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct ggcase GGML_GLU_OP_SWIGLU:ggml_cann_swiglu(ctx, dst);break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ ggml_cann_swiglu_clamp(ctx, dst);+ break;case GGML_GLU_OP_GEGLU_QUICK:ggml_cann_geglu_quick(ctx, dst);break;@@ -2428,6 +2431,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_tencase GGML_GLU_OP_SWIGLU:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:return true;default:return false;diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cindex b9c0fa3ddc..6bc4467e37 100644--- a/ggml/src/ggml-cpu/ggml-cpu.c+++ b/ggml/src/ggml-cpu/ggml-cpu.c@@ -2311,6 +2311,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {case GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:{n_tasks = n_threads;} break;diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex b47ce5463c..266261c5e5 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -3403,6 +3403,139 @@ static void ggml_compute_forward_swiglu_oai(}}+// ggml_compute_forward_swiglu_clamp++static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * params, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ char * src0_d = (char *) src0->data;+ char * src1_d = (char *) (src1 ? src1->data : src0->data);+ const size_t src0_o = src0->nb[1];+ const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];++ GGML_ASSERT(ggml_is_contiguous_1(src0));+ GGML_ASSERT(ggml_is_contiguous_1(dst));++ if (src1) {+ GGML_ASSERT(ggml_is_contiguous_1(src1));+ GGML_ASSERT(src0->type == src1->type);+ }++ const int ith = params->ith;+ const int nth = params->nth;++ const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;+ const int nr = ggml_nrows(src0);++ GGML_ASSERT(dst->ne[0] == nc);+ GGML_ASSERT(ggml_nrows(dst) == nr);++ const int32_t swapped = ggml_get_op_params_i32(dst, 1);+ const float limit = ggml_get_op_params_f32(dst, 3);++ const int dr = (nr + nth - 1) / nth;+ const int ir0 = dr * ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int i1 = ir0; i1 < ir1; i1++) {+ float * src0_p = (float *) (src0_d + i1 * src0_o);+ float * src1_p = (float *) (src1_d + i1 * src1_o);+ float * dst_p = (float *) ((char *) dst->data + i1 * (dst->nb[1]));++ if (!src1) {+ src0_p += swapped ? nc : 0;+ src1_p += swapped ? 0 : nc;+ }++ for (int k = 0; k < nc; k++) {+ const float gate = std::min(src0_p[k], limit);+ const float up = std::clamp(src1_p[k], -limit, limit);+ dst_p[k] = gate / (1.f + expf(-gate)) * up;+ }++#ifndef NDEBUG+ for (int k = 0; k < nc; k++) {+ const float x = dst_p[k];+ GGML_UNUSED(x);+ assert(!isnan(x));+ assert(!isinf(x));+ }+#endif // NDEBUG+ }+}++static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ char * src0_d = (char *) src0->data;+ char * src1_d = (char *) (src1 ? src1->data : src0->data);+ const size_t src0_o = src0->nb[1];+ const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];++ GGML_ASSERT(ggml_is_contiguous_1(src0));+ GGML_ASSERT(ggml_is_contiguous_1(dst));++ if (src1) {+ GGML_ASSERT(ggml_is_contiguous_1(src1));+ GGML_ASSERT(src0->type == src1->type);+ }++ const int ith = params->ith;+ const int nth = params->nth;++ const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;+ const int nr = ggml_nrows(src0);++ GGML_ASSERT(dst->ne[0] == nc);+ GGML_ASSERT(ggml_nrows(dst) == nr);++ const int32_t swapped = ggml_get_op_params_i32(dst, 1);+ const float limit = ggml_get_op_params_f32(dst, 3);++ const int dr = (nr + nth - 1) / nth;+ const int ir0 = dr * ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int i1 = ir0; i1 < ir1; i1++) {+ ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o);+ ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o);+ ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1]));++ if (!src1) {+ src0_p += swapped ? nc : 0;+ src1_p += swapped ? 0 : nc;+ }++ for (int k = 0; k < nc; k++) {+ const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit);+ const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit);+ dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up);+ }++#ifndef NDEBUG+ for (int k = 0; k < nc; k++) {+ const float x = GGML_FP16_TO_FP32(dst_p[k]);+ GGML_UNUSED(x);+ assert(!isnan(x));+ assert(!isinf(x));+ }+#endif // NDEBUG+ }+}++static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params, ggml_tensor * dst) {+ switch (dst->src[0]->type) {+ case GGML_TYPE_F32:+ ggml_compute_forward_swiglu_clamp_f32(params, dst);+ break;+ case GGML_TYPE_F16:+ ggml_compute_forward_swiglu_clamp_f16(params, dst);+ break;+ default:+ GGML_ABORT("fatal error");+ }+}+// ggml_compute_forward_geglu_erfstatic void ggml_compute_forward_geglu_erf_f32(@@ -10136,6 +10269,10 @@ void ggml_compute_forward_glu({ggml_compute_forward_geglu_quick(params, dst);} break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ {+ ggml_compute_forward_swiglu_clamp(params, dst);+ } break;default:{GGML_ABORT("fatal error");diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuhindex 14dd1098c9..e5ccd1feab 100644--- a/ggml/src/ggml-cuda/common.cuh+++ b/ggml/src/ggml-cuda/common.cuh@@ -1539,6 +1539,7 @@ struct ggml_cuda_mm_fusion_args_host {const ggml_tensor * x_scale = nullptr;const ggml_tensor * gate_scale = nullptr;ggml_glu_op glu_op;+ float glu_limit = 0.0f;};struct ggml_cuda_mm_fusion_args_device {const void * x_bias = nullptr;@@ -1547,6 +1548,7 @@ struct ggml_cuda_mm_fusion_args_device {const void * x_scale = nullptr;const void * gate_scale = nullptr;ggml_glu_op glu_op;+ float glu_limit = 0.0f;};struct ggml_cuda_kernel_launch_params {@@ -1673,4 +1675,3 @@ static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_kekernel<<<launch_params.block_nums, launch_params.block_dims, launch_params.shmem, launch_params.stream>>>(std::forward<Args>(args)... );CUDA_CHECK(cudaGetLastError());}-diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex bd9754c2ff..7c2d3cad20 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -1744,7 +1744,7 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up,return false;}- static constexpr std::array<ggml_glu_op, 3> valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI };+ static constexpr std::array<ggml_glu_op, 4> valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI, GGML_GLU_OP_SWIGLU_CLAMP };if (std::find(valid_glu_ops.begin(), valid_glu_ops.end(), ggml_get_glu_op(glu)) == valid_glu_ops.end()) {return false;@@ -2203,6 +2203,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggcase GGML_GLU_OP_GEGLU_QUICK:ggml_cuda_op_geglu_quick(ctx, dst);break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ ggml_cuda_op_swiglu_clamp(ctx, dst);+ break;default:return false;}@@ -3595,6 +3598,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphfusion_data.x_scale = up_scale;fusion_data.gate_scale = gate_scale;fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);@@ -3688,6 +3692,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphfusion_data.x_scale = up_scale;fusion_data.gate_scale = gate_scale;fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);@@ -3744,6 +3749,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphfusion_data.x_bias = up_bias_tensor;fusion_data.gate_bias = gate_bias_tensor;fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data);fused_mul_mat_vec = true;@@ -3757,6 +3763,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphfusion_data.x_bias = up_bias_tensor;fusion_data.gate_bias = gate_bias_tensor;fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data);fused_mul_mat_vec = true;@@ -3781,8 +3788,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphif (ggml_cuda_should_fuse_mul_mat_vec_f(up)) {ggml_cuda_mm_fusion_args_host fusion_data{};- fusion_data.gate = gate->src[0];- fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.gate = gate->src[0];+ fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data);fused_mul_mat_vec = true;@@ -3792,8 +3800,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphif (ggml_cuda_should_fuse_mul_mat_vec_q(up)) {ggml_cuda_mm_fusion_args_host fusion_data{};- fusion_data.gate = gate->src[0];- fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.gate = gate->src[0];+ fusion_data.glu_op = ggml_get_glu_op(glu);+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data);fused_mul_mat_vec = true;@@ -4919,6 +4928,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:return ggml_is_contiguous_1(op->src[0]);default:return false;diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cuindex d7dbc8b992..bd5c5d421a 100644--- a/ggml/src/ggml-cuda/mmvf.cu+++ b/ggml/src/ggml-cuda/mmvf.cu@@ -56,6 +56,7 @@ static __global__ void mul_mat_vec_f(bool use_bias = false;bool use_gate_bias = false;ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU;+ float glu_limit = 0.0f;const T * gate_x = nullptr;const float * x_bias = nullptr;const float * gate_bias = nullptr;@@ -65,6 +66,7 @@ static __global__ void mul_mat_vec_f(use_bias = fusion.x_bias != nullptr;use_gate_bias = fusion.gate_bias != nullptr;glu_op = fusion.glu_op;+ glu_limit = fusion.glu_limit;if (use_gate) {gate_x = static_cast<const T *>(fusion.gate);@@ -365,6 +367,9 @@ static __global__ void mul_mat_vec_f(value = ggml_cuda_op_swiglu_oai_single(gate_value, value);break;}+ case GGML_GLU_OP_SWIGLU_CLAMP:+ value = ggml_cuda_op_swiglu_clamp_single(gate_value, value, glu_limit);+ break;default:break;}@@ -374,7 +379,7 @@ static __global__ void mul_mat_vec_f(dst[tid*stride_col_dst + row] = value;if constexpr (!has_fusion) {- GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, gate_x, x_bias, gate_bias, sumf_gate);+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, glu_limit, gate_x, x_bias, gate_bias, sumf_gate);}}@@ -675,6 +680,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensorfusion_local.gate_bias = fusion->gate_bias->data;}fusion_local.glu_op = fusion->glu_op;+ fusion_local.glu_limit = fusion->glu_limit;}const int64_t s01 = src0->nb[1] / ts_src0;diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cuindex 9705348098..79f7a3f6fe 100644--- a/ggml/src/ggml-cuda/mmvq.cu+++ b/ggml/src/ggml-cuda/mmvq.cu@@ -595,6 +595,7 @@ static __global__ void mul_mat_vec_q(const float * x_scale = nullptr;const float * gate_scale = nullptr;ggml_glu_op active_glu;+ float glu_limit = 0.0f;if constexpr (has_fusion) {use_gate = fusion.gate != nullptr;@@ -604,6 +605,7 @@ static __global__ void mul_mat_vec_q(x_bias = (const float *) fusion.x_bias;gate_bias = (const float *) fusion.gate_bias;active_glu = fusion.glu_op;+ glu_limit = fusion.glu_limit;if constexpr (type == GGML_TYPE_NVFP4) {use_scale = fusion.x_scale != nullptr;use_gate_scale = fusion.gate_scale != nullptr && use_gate;@@ -745,6 +747,9 @@ static __global__ void mul_mat_vec_q(case GGML_GLU_OP_SWIGLU_OAI:result = ggml_cuda_op_swiglu_oai_single(gate_value, result);break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);+ break;default:result = result * gate_value;break;@@ -757,7 +762,7 @@ static __global__ void mul_mat_vec_q(}if constexpr (!has_fusion) {- GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, glu_limit, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);}if constexpr (type != GGML_TYPE_NVFP4) {GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales);@@ -1310,6 +1315,7 @@ void ggml_cuda_mul_mat_vec_q(fusion_local.gate_scale = fusion->gate_scale->data;}fusion_local.glu_op = fusion->glu_op;+ fusion_local.glu_limit = fusion->glu_limit;}// If src0 is a temporary compute buffer, clear any potential padding.diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cuindex 4cb805fa60..d3e594878f 100644--- a/ggml/src/ggml-cuda/unary.cu+++ b/ggml/src/ggml-cuda/unary.cu@@ -427,6 +427,81 @@ void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst)swiglu_oai_cuda(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream);}+// swiglu_clamp++template <typename T>+static __global__ void swiglu_clamp_kernel(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, float limit) {+ const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;+ }++ const int64_t j0 = (i / n) * o0 + (i % n);+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);++ dst[i] = (T) ggml_cuda_op_swiglu_clamp_single((float) gate[j0], (float) up[j1], limit);+}++template <typename T>+static void swiglu_clamp_cuda(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, const float limit, cudaStream_t stream) {+ const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE;+ swiglu_clamp_kernel<<<num_blocks, CUDA_GLU_BLOCK_SIZE, 0, stream>>>(gate, up, dst, k, n, o0, o1, limit);+}++void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ void * src0_d = src0->data;+ void * src1_d = src1 ? src1->data : src0->data;+ const int64_t src0_o = src0->nb[1];+ const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];+ void * dst_d = dst->data;+ const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;+ cudaStream_t stream = ctx.stream();++ GGML_ASSERT(ggml_is_contiguous_1(src0));+ GGML_ASSERT(src0->nb[0] == ggml_element_size(src0));+ GGML_ASSERT(ggml_is_contiguous(dst));++ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src0->type == dst->type);+ GGML_ASSERT(dst->ne[0] == nc);+ GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0));++ if (src1) {+ GGML_ASSERT(ggml_is_contiguous_1(src1));+ GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));+ GGML_ASSERT(src1->ne[0] == nc);+ GGML_ASSERT(src0->type == src1->type);+ }++ const int32_t swapped = ggml_get_op_params_i32(dst, 1);+ const float limit = ggml_get_op_params_f32(dst, 3);++ if (src0->type == GGML_TYPE_F16) {+ half * src0_p = (half *) src0_d;+ half * src1_p = (half *) src1_d;++ if (!src1) {+ src0_p += swapped ? nc : 0;+ src1_p += swapped ? 0 : nc;+ }++ swiglu_clamp_cuda(src0_p, src1_p, (half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(half), src1_o / sizeof(half), limit, stream);+ } else {+ float * src0_p = (float *) src0_d;+ float * src1_p = (float *) src1_d;++ if (!src1) {+ src0_p += swapped ? nc : 0;+ src1_p += swapped ? 0 : nc;+ }++ swiglu_clamp_cuda(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), limit, stream);+ }+}+/* CUDA kernel + launcher for xIELU */template <typename T>diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuhindex 81ed873ecc..04f3af6443 100644--- a/ggml/src/ggml-cuda/unary.cuh+++ b/ggml/src/ggml-cuda/unary.cuh@@ -83,6 +83,8 @@ void ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);void ggml_cuda_op_geglu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);@@ -112,3 +114,10 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_oai_single(float x, float gout_glu = out_glu * (1.0f + g);return out_glu;}++__device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, float up, float limit) {+ gate = fminf(gate, limit);+ up = fmaxf(fminf(up, limit), -limit);++ return ggml_cuda_op_silu_single(gate) * up;+}diff --git a/ggml/src/ggml-et/et-kernels/src/glu_f32.c b/ggml/src/ggml-et/et-kernels/src/glu_f32.cindex 95fe572158..d376d6f56f 100644--- a/ggml/src/ggml-et/et-kernels/src/glu_f32.c+++ b/ggml/src/ggml-et/et-kernels/src/glu_f32.c@@ -17,7 +17,7 @@ struct ggml_et_glu_params {int32_t glu_op_type; // GLU operation type (REGLU=0, GEGLU=1, SWIGLU=2, etc.)int32_t swapped; // Whether gate and value are swappedfloat alpha; // SWIGLU_OAI: sigmoid scaling factor- float limit; // SWIGLU_OAI: clamp limit+ float limit; // GLU clamp limit};// SiLU activation function: silu(x) = x * sigmoid(x) = x / (1 + exp(-x))@@ -332,6 +332,57 @@ static inline void block_swiglu_oai(float * dst_block,}}+static inline void block_swiglu_clamp(float * dst_block,+ const float * gate_block,+ const float * up_block,+ int elements,+ float limit) {+ int32_t vec_end = (elements / 8) * 8;++ unsigned long temp_mask;+ __asm__ volatile("mova.x.m %0" : "=r"(temp_mask));+ __asm__ volatile("mov.m.x m0, x0, 0xFF");++ float one_const = 1.0f;+ float limit_pos = limit;+ float limit_neg = -limit;+ float neg_log2e = -1.4426950408889634f;++ for (int32_t i = 0; i < vec_end; i += 8) {+ __asm__ volatile(+ "flw.ps f10, %[gate_vec]\n"+ "flw.ps f11, %[up_vec]\n"+ "fbc.ps f21, %[one_ptr]\n"+ "fbc.ps f23, %[lim_pos]\n"+ "fbc.ps f24, %[lim_neg]\n"+ "fbc.ps f25, %[k_ptr]\n"+ "fmin.ps f12, f10, f23\n"+ "fmax.ps f13, f11, f24\n"+ "fmin.ps f13, f13, f23\n"+ "fmul.ps f14, f12, f25\n"+ "fexp.ps f15, f14\n"+ "fadd.ps f15, f15, f21\n"+ "frcp.ps f16, f15\n"+ "fmul.ps f17, f12, f16\n"+ "fmul.ps f18, f17, f13\n"+ "fsw.ps f18, %[dst_out]\n"+ : [dst_out] "=m"(*(float (*)[8]) & dst_block[i])+ : [gate_vec] "m"(*(const float (*)[8]) & gate_block[i]), [up_vec] "m"(*(const float (*)[8]) & up_block[i]),+ [one_ptr] "m"(one_const), [lim_pos] "m"(limit_pos), [lim_neg] "m"(limit_neg), [k_ptr] "m"(neg_log2e)+ : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f21", "f23", "f24", "f25");+ }++ __asm__ volatile("mova.m.x %0" :: "r"(temp_mask));++ for (int32_t i = vec_end; i < elements; i++) {+ float gate = gate_block[i] > limit ? limit : gate_block[i];+ float up = up_block[i];+ up = up > limit ? limit : up;+ up = up < -limit ? -limit : up;+ dst_block[i] = silu_f32(gate) * up;+ }+}+// Scalar erf approximation (Abramowitz & Stegun 7.1.26, max error ~1.5e-7)static inline float erf_approx(float x) {const float a1 = 0.254829592f;@@ -386,6 +437,7 @@ int entry_point(struct ggml_et_glu_params * params, void * env) {switch (params->glu_op_type) {case GGML_GLU_OP_SWIGLU:case GGML_GLU_OP_SWIGLU_OAI:+ case GGML_GLU_OP_SWIGLU_CLAMP:case GGML_GLU_OP_GEGLU:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:@@ -531,6 +583,9 @@ int entry_point(struct ggml_et_glu_params * params, void * env) {case GGML_GLU_OP_SWIGLU_OAI:block_swiglu_oai(dst_ptr, x_ptr, g_ptr, (int) elements_to_process, params->alpha, params->limit);break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ block_swiglu_clamp(dst_ptr, x_ptr, g_ptr, (int) elements_to_process, params->limit);+ break;default:return -1;}diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp b/ggml/src/ggml-et/ggml-et-cpu-compare.cppindex b37f6d261d..5771679b3f 100644--- a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp+++ b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp@@ -261,7 +261,12 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ctGGML_LOG_ERROR("ET: GLU CPU comparison requires split tensor mode\n");return false;}- ctx->cpu_dst = ggml_glu_split(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, glu_op);+ if (glu_op == GGML_GLU_OP_SWIGLU_CLAMP) {+ const float limit = ggml_get_op_params_f32(node, 3);+ ctx->cpu_dst = ggml_swiglu_clamp(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, limit);+ } else {+ ctx->cpu_dst = ggml_glu_split(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, glu_op);+ }}break;case GGML_OP_SOFT_MAX:diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cppindex 7871d52408..8765138672 100644--- a/ggml/src/ggml-et/ggml-et-ops.cpp+++ b/ggml/src/ggml-et/ggml-et-ops.cpp@@ -636,6 +636,7 @@ bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensorcase GGML_GLU_OP_GEGLU:case GGML_GLU_OP_SWIGLU:case GGML_GLU_OP_SWIGLU_OAI:+ case GGML_GLU_OP_SWIGLU_CLAMP:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:break;@@ -661,6 +662,8 @@ bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensorparams.limit = 0.0f;if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI) {params.alpha = ggml_get_op_params_f32(node, 2);+ }+ if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI || glu_op_type == GGML_GLU_OP_SWIGLU_CLAMP) {params.limit = ggml_get_op_params_f32(node, 3);}// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cppindex b87b189a57..61c31d6f29 100644--- a/ggml/src/ggml-et/ggml-et.cpp+++ b/ggml/src/ggml-et/ggml-et.cpp@@ -1210,7 +1210,8 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm// Check GLU variant - support SWIGLU, SWIGLU_OAI, GEGLU, GEGLU_ERF, GEGLU_QUICK, REGLUggml_glu_op glu_type = ggml_get_glu_op(op);const bool supported_variant = glu_type == GGML_GLU_OP_SWIGLU || glu_type == GGML_GLU_OP_SWIGLU_OAI ||- glu_type == GGML_GLU_OP_GEGLU || glu_type == GGML_GLU_OP_GEGLU_ERF ||+ glu_type == GGML_GLU_OP_SWIGLU_CLAMP || glu_type == GGML_GLU_OP_GEGLU ||+ glu_type == GGML_GLU_OP_GEGLU_ERF ||glu_type == GGML_GLU_OP_GEGLU_QUICK || glu_type == GGML_GLU_OP_REGLU;if (op->src[1]) {diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 87e6989bd2..3eb84fd2a9 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -4701,6 +4701,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {switch (ggml_get_glu_op(t)) {case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;+ case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;default: break;}@@ -5528,6 +5529,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, consswitch (ggml_get_glu_op(op)) {case GGML_GLU_OP_SWIGLU:case GGML_GLU_OP_SWIGLU_OAI:+ case GGML_GLU_OP_SWIGLU_CLAMP:case GGML_GLU_OP_GEGLU:supp = ggml_hexagon_supported_activations(sess, op);break;diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.cindex 0a8bf84e38..ac00b447d9 100644--- a/ggml/src/ggml-hexagon/htp/act-ops.c+++ b/ggml/src/ggml-hexagon/htp/act-ops.c@@ -180,6 +180,26 @@ static void swiglu_oai_f32(const float * restrict src0,}}+static void swiglu_clamp_f32(const float * restrict src0,+ const float * restrict src1,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;+ const float limit = ((const float *) (actx->octx->op_params))[3];++ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);++ hvx_min_scalar_f32((uint8_t *) src0_ptr, src0_ptr, limit, nc);+ hvx_clamp_scalar_f32((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);+ hvx_sigmoid_f32_aa(dst_ptr, src0_ptr, nc);+ hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);+ }+}+static const float GELU_COEF_A = 0.044715f;static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;@@ -411,6 +431,7 @@ static void geglu_f32(const float * restrict src0,DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx))DEFINE_GLU_PER_THREAD(swiglu_oai, "swiglu-oai-f32", swiglu_oai_f32(src0_spad, src1_spad, dst_spad, block_size, actx))+DEFINE_GLU_PER_THREAD(swiglu_clamp, "swiglu-clamp-f32", swiglu_clamp_f32(src0_spad, src1_spad, dst_spad, block_size, actx))DEFINE_GLU_PER_THREAD(geglu, "geglu-f32", geglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx))static int execute_op_activations_f32(struct htp_ops_context * octx) {@@ -437,6 +458,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {op_type = "swiglu-oai-f32";break;+ case HTP_OP_GLU_SWIGLU_CLAMP:+ act_op_func = (worker_callback_t) glu_swiglu_clamp_f32_per_thread;+ op_type = "swiglu-clamp-f32";+ break;+case HTP_OP_GLU_GEGLU:act_op_func = (worker_callback_t)glu_geglu_f32_per_thread;op_type = "geglu-f32";@@ -527,7 +553,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {const uint8_t * data_src0 = (const uint8_t *) src0->data;const uint8_t * data_src1 = src1 ? (const uint8_t *) src1->data : NULL;- if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_GEGLU)) {+ if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_SWIGLU_CLAMP || octx->op == HTP_OP_GLU_GEGLU)) {const int32_t swapped = octx->op_params[1];data_src1 = data_src0;actx.src1_row_size = actx.src0_row_size;diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.hindex e804844d59..53c95f28d3 100644--- a/ggml/src/ggml-hexagon/htp/htp-ops.h+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h@@ -96,6 +96,7 @@ enum htp_op_code {HTP_OP_FENCE,HTP_OP_ALLREDUCE,HTP_OP_ALLREDUCE_ADD,+ HTP_OP_GLU_SWIGLU_CLAMP,HTP_OP_INVALID};diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex fe7d093a81..27d1dedcdf 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -784,6 +784,7 @@ static int execute_op(struct htp_ops_context * octx) {case HTP_OP_GLU_SWIGLU:case HTP_OP_GLU_SWIGLU_OAI:+ case HTP_OP_GLU_SWIGLU_CLAMP:case HTP_OP_GLU_GEGLU:return op_activations(octx);diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex 4e855be446..5a2f01f1db 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -318,6 +318,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu(ggml_metal_lcase GGML_GLU_OP_SWIGLU_OAI: op_str = "swiglu_oai"; break;case GGML_GLU_OP_GEGLU_ERF: op_str = "geglu_erf"; break;case GGML_GLU_OP_GEGLU_QUICK: op_str = "geglu_quick"; break;+ case GGML_GLU_OP_SWIGLU_CLAMP: op_str = "swiglu_clamp"; break;default: GGML_ABORT("fatal error");} break;default: GGML_ABORT("fatal error");diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex a053887a33..83344539e5 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -1510,6 +1510,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tecase GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:return ggml_is_contiguous_1(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);default:return false;diff --git a/ggml/src/ggml-metal/kernels/unary.metal b/ggml/src/ggml-metal/kernels/unary.metalindex 39cad0cbee..e50a648639 100644--- a/ggml/src/ggml-metal/kernels/unary.metal+++ b/ggml/src/ggml-metal/kernels/unary.metal@@ -317,6 +317,32 @@ typedef decltype(kernel_swiglu_oai<float>) kernel_swiglu_oai_t;template [[host_name("kernel_swiglu_oai_f32")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai<float>;template [[host_name("kernel_swiglu_oai_f16")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai<half>;+template<typename T>+kernel void kernel_swiglu_clamp(+ constant ggml_metal_kargs_glu & args,+ device const char * src0,+ device const char * src1,+ device char * dst,+ uint tgpig[[threadgroup_position_in_grid]],+ uint tpitg[[thread_position_in_threadgroup]],+ uint ntg[[threads_per_threadgroup]]) {+ device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;+ device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;+ device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);++ for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {+ const float gate = min((float) src0_row[i0], args.limit);+ const float up = clamp((float) src1_row[i0], -args.limit, args.limit);++ dst_row[i0] = (T)(gate / (1.0f + exp(-gate)) * up);+ }+}++typedef decltype(kernel_swiglu_clamp<float>) kernel_swiglu_clamp_t;++template [[host_name("kernel_swiglu_clamp_f32")]] kernel kernel_swiglu_clamp_t kernel_swiglu_clamp<float>;+template [[host_name("kernel_swiglu_clamp_f16")]] kernel kernel_swiglu_clamp_t kernel_swiglu_clamp<half>;+template<typename T>kernel void kernel_geglu_erf(constant ggml_metal_kargs_glu & args,diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 426aac5231..90635cc858 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -744,8 +744,9 @@ struct ggml_backend_opencl_context {cl_kernel kernel_tri;cl_kernel kernel_fill;cl_kernel kernel_clamp;- cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_geglu_erf, kernel_geglu_quick,- kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_geglu_erf_f16, kernel_geglu_quick_f16;+ cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf,+ kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16,+ kernel_geglu_erf_f16, kernel_geglu_quick_f16;cl_kernel kernel_norm, kernel_norm_mul_add;cl_kernel kernel_rms_norm, kernel_rms_norm_mul;cl_kernel kernel_l2_norm_f32;@@ -1601,11 +1602,13 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {CL_CHECK((backend_ctx->kernel_reglu = clCreateKernel(backend_ctx->program_glu, "kernel_reglu", &err), err));CL_CHECK((backend_ctx->kernel_swiglu = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu", &err), err));CL_CHECK((backend_ctx->kernel_swiglu_oai = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_oai", &err), err));+ CL_CHECK((backend_ctx->kernel_swiglu_clamp = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp", &err), err));CL_CHECK((backend_ctx->kernel_geglu_erf = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf", &err), err));CL_CHECK((backend_ctx->kernel_geglu_quick = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick", &err), err));CL_CHECK((backend_ctx->kernel_geglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_f16", &err), err));CL_CHECK((backend_ctx->kernel_reglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_reglu_f16", &err), err));CL_CHECK((backend_ctx->kernel_swiglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_f16", &err), err));+ CL_CHECK((backend_ctx->kernel_swiglu_clamp_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp_f16", &err), err));CL_CHECK((backend_ctx->kernel_geglu_erf_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf_f16", &err), err));CL_CHECK((backend_ctx->kernel_geglu_quick_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick_f16", &err), err));GGML_LOG_CONT(".");@@ -7700,6 +7703,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tecase GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);default:return false;@@ -24886,6 +24890,13 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, constcase GGML_GLU_OP_SWIGLU_OAI:kernel = backend_ctx->kernel_swiglu_oai;break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ if (dst->type == GGML_TYPE_F32) {+ kernel = backend_ctx->kernel_swiglu_clamp;+ } else {+ kernel = backend_ctx->kernel_swiglu_clamp_f16;+ }+ break;case GGML_GLU_OP_GEGLU_ERF:if (dst->type == GGML_TYPE_F32) {kernel = backend_ctx->kernel_geglu_erf;@@ -24941,8 +24952,10 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, constCL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00_off));CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10_off));- if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) {+ if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_CLAMP) {CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &limit));+ }+ if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) {CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &alpha));}diff --git a/ggml/src/ggml-opencl/kernels/glu.cl b/ggml/src/ggml-opencl/kernels/glu.clindex 059a4bbf1b..30bad00f7d 100644--- a/ggml/src/ggml-opencl/kernels/glu.cl+++ b/ggml/src/ggml-opencl/kernels/glu.cl@@ -243,6 +243,71 @@ kernel void kernel_swiglu_oai(}}+//------------------------------------------------------------------------------+// swiglu_clamp+//------------------------------------------------------------------------------+kernel void kernel_swiglu_clamp(+ global char * src0,+ ulong offset0,+ global char * src1,+ ulong offset1,+ global char * dst,+ ulong offsetd,+ ulong nb01,+ ulong nb11,+ int ne0,+ ulong nb1,+ int ne00_off,+ int ne10_off,+ float limit+) {+ src0 = (global char*)((global char*)src0 + offset0);+ src1 = (global char*)((global char*)src1 + offset1);+ dst = (global char*)((global char*)dst + offsetd);++ global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;+ global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;+ global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);++ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {+ const float gate = min(src0_row[i0], limit);+ const float up = clamp(src1_row[i0], -limit, limit);++ dst_row[i0] = gate / (1.0f + exp(-gate)) * up;+ }+}++kernel void kernel_swiglu_clamp_f16(+ global char * src0,+ ulong offset0,+ global char * src1,+ ulong offset1,+ global char * dst,+ ulong offsetd,+ ulong nb01,+ ulong nb11,+ int ne0,+ ulong nb1,+ int ne00_off,+ int ne10_off,+ float limit+) {+ src0 = (global char*)((global char*)src0 + offset0);+ src1 = (global char*)((global char*)src1 + offset1);+ dst = (global char*)((global char*)dst + offsetd);++ global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;+ global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;+ global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1);++ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {+ const float gate = min((float) src0_row[i0], limit);+ const float up = clamp((float) src1_row[i0], -limit, limit);++ dst_row[i0] = (half) (gate / (1.0f + exp(-gate)) * up);+ }+}+//------------------------------------------------------------------------------// geglu_erf//------------------------------------------------------------------------------diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cppindex d220f2f584..d81fc53b5d 100644--- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp+++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp@@ -89,6 +89,21 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {return rename_outputs_with_suffix({res}, context.get_name());}+OutputVector translate_glu_swiglu_clamp(const NodeContext & context) {+ auto [src0, src1] = get_glu_inputs(context);++ const int32_t * params = context.get_output_op_params();+ const float limit = reinterpret_cast<const float *>(params)[3];++ auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);+ auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(gate);+ auto silu = std::make_shared<ov::op::v1::Multiply>(gate, sigmoid);+ auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);+ auto res = std::make_shared<ov::op::v1::Multiply>(silu, up);++ return rename_outputs_with_suffix({res}, context.get_name());+}+} // namespace op} // namespace ggml} // namespace frontenddiff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cppindex 9c9d8eeac7..d4f5ac3073 100644--- a/ggml/src/ggml-openvino/openvino/op_table.cpp+++ b/ggml/src/ggml-openvino/openvino/op_table.cpp@@ -60,6 +60,7 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {{"GGML_OP_VIEW", op::translate_view },{"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu },{"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai },+ {"GGML_GLU_OP_SWIGLU_CLAMP", op::translate_glu_swiglu_clamp },{"GGML_GLU_OP_GEGLU", op::translate_glu_geglu },{"GGML_GLU_OP_GEGLU_QUICK", op::translate_glu_geglu_quick },{"GGML_OP_SET_ROWS", op::translate_set_rows },diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.hindex 0a81a57a66..a0a42bff33 100644--- a/ggml/src/ggml-openvino/openvino/op_table.h+++ b/ggml/src/ggml-openvino/openvino/op_table.h@@ -37,6 +37,7 @@ GGML_OP_CONVERTER(translate_transpose);GGML_OP_CONVERTER(translate_view);GGML_OP_CONVERTER(translate_glu_swiglu);GGML_OP_CONVERTER(translate_glu_swiglu_oai);+GGML_OP_CONVERTER(translate_glu_swiglu_clamp);GGML_OP_CONVERTER(translate_glu_geglu);GGML_OP_CONVERTER(translate_glu_geglu_quick);GGML_OP_CONVERTER(translate_set_rows);diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cppindex 95914873e5..2e926abea7 100644--- a/ggml/src/ggml-sycl/element_wise.cpp+++ b/ggml/src/ggml-sycl/element_wise.cpp@@ -1132,6 +1132,102 @@ void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst)swiglu_oai_sycl(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream);}+template <typename T>+static void swiglu_clamp_kernel(const T * gate,+ const T * up,+ T * dst,+ const int64_t k,+ const int64_t n,+ const int64_t o0,+ const int64_t o1,+ float limit,+ sycl::nd_item<3> item_ct1) {+ const int64_t i = int64_t(item_ct1.get_local_range(2)) * item_ct1.get_group(2) + item_ct1.get_local_id(2);++ if (i >= k) {+ return;+ }++ const int64_t j0 = (i / n) * o0 + (i % n);+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);++ const float gate_value = sycl::fmin((float) gate[j0], limit);+ const float up_value = sycl::fmax(sycl::fmin((float) up[j1], limit), -limit);+ dst[i] = (T) (gate_value / (1.0f + sycl::native::exp(-gate_value)) * up_value);+}++template <typename T>+static void swiglu_clamp_sycl(const T * gate,+ const T * up,+ T * dst,+ const int64_t k,+ const int64_t n,+ const int64_t o0,+ const int64_t o1,+ float limit,+ dpct::queue_ptr stream) {+ const int64_t num_blocks = (k + SYCL_GLU_BLOCK_SIZE - 1) / SYCL_GLU_BLOCK_SIZE;+ stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE),+ sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE)),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ swiglu_clamp_kernel(gate, up, dst, k, n, o0, o1, limit, item_ct1);+ });+}++static void ggml_sycl_op_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ void * src0_d = src0->data;+ void * src1_d = src1 ? src1->data : src0->data;+ const int64_t src0_o = src0->nb[1];+ const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];+ void * dst_d = dst->data;+ const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;+ dpct::queue_ptr stream = ctx.stream();++ GGML_ASSERT(ggml_is_contiguous_1(src0));+ GGML_ASSERT(src0->nb[0] == ggml_element_size(src0));+ GGML_ASSERT(ggml_is_contiguous(dst));+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src0->type == dst->type);+ GGML_ASSERT(dst->ne[0] == nc);+ GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0));++ if (src1) {+ GGML_ASSERT(ggml_is_contiguous_1(src1));+ GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));+ GGML_ASSERT(src1->ne[0] == nc);+ GGML_ASSERT(src0->type == src1->type);+ }++ const int32_t swapped = ggml_get_op_params_i32(dst, 1);+ const float limit = ggml_get_op_params_f32(dst, 3);++ if (src0->type == GGML_TYPE_F16) {+ sycl::half * src0_p = (sycl::half *) src0_d;+ sycl::half * src1_p = (sycl::half *) src1_d;++ if (!src1) {+ src0_p += swapped ? nc : 0;+ src1_p += swapped ? 0 : nc;+ }++ swiglu_clamp_sycl(src0_p, src1_p, (sycl::half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(sycl::half),+ src1_o / sizeof(sycl::half), limit, stream);+ } else {+ float * src0_p = (float *) src0_d;+ float * src1_p = (float *) src1_d;++ if (!src1) {+ src0_p += swapped ? nc : 0;+ src1_p += swapped ? 0 : nc;+ }++ swiglu_clamp_sycl(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float),+ src1_o / sizeof(float), limit, stream);+ }+}+static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {return op_gelu_erf(x);@@ -1295,6 +1391,11 @@ void ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {ggml_sycl_op_swiglu_oai(ctx, dst);}+void ggml_sycl_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);+ ggml_sycl_op_swiglu_clamp(ctx, dst);+}+void ggml_sycl_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);ggml_sycl_op_geglu_erf(ctx, dst);diff --git a/ggml/src/ggml-sycl/element_wise.hpp b/ggml/src/ggml-sycl/element_wise.hppindex 67bf422d2f..d280066efb 100644--- a/ggml/src/ggml-sycl/element_wise.hpp+++ b/ggml/src/ggml-sycl/element_wise.hpp@@ -77,6 +77,7 @@ void ggml_sycl_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);void ggml_sycl_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst);void ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst);+void ggml_sycl_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst);void ggml_sycl_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst);diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cppindex d58ffd00da..2b2d26cf23 100644--- a/ggml/src/ggml-sycl/ggml-sycl.cpp+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp@@ -5373,6 +5373,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggcase GGML_GLU_OP_SWIGLU_OAI:ggml_sycl_swiglu_oai(ctx, dst);break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ ggml_sycl_swiglu_clamp(ctx, dst);+ break;case GGML_GLU_OP_GEGLU_ERF:ggml_sycl_geglu_erf(ctx, dst);break;@@ -6133,6 +6136,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, conscase GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:return ggml_is_contiguous_1(op->src[0]);default:return false;diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 8fbb1359f4..394c84bf25 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -1035,6 +1035,7 @@ struct vk_device_struct {vk_pipeline pipeline_reglu[2];vk_pipeline pipeline_swiglu[2];vk_pipeline pipeline_swiglu_oai[2];+ vk_pipeline pipeline_swiglu_clamp[2];vk_pipeline pipeline_geglu_erf[2];vk_pipeline pipeline_geglu_quick[2];@@ -5748,6 +5749,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {CREATE_GLU(reglu)CREATE_GLU(swiglu)CREATE_GLU(swiglu_oai)+ CREATE_GLU(swiglu_clamp)CREATE_GLU(geglu_erf)CREATE_GLU(geglu_quick)#undef CREATE_GLU@@ -11578,6 +11580,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, constreturn ctx->device->pipeline_swiglu[dst->type == GGML_TYPE_F16];case GGML_GLU_OP_SWIGLU_OAI:return ctx->device->pipeline_swiglu_oai[dst->type == GGML_TYPE_F16];+ case GGML_GLU_OP_SWIGLU_CLAMP:+ return ctx->device->pipeline_swiglu_clamp[dst->type == GGML_TYPE_F16];case GGML_GLU_OP_GEGLU_ERF:return ctx->device->pipeline_geglu_erf[dst->type == GGML_TYPE_F16];case GGML_GLU_OP_GEGLU_QUICK:@@ -15883,6 +15887,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgrcase GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:ggml_vk_glu(ctx, compute_ctx, src0, src1, node);break;default:@@ -18400,6 +18405,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmcase GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&(op->src[0]->type == op->type) &&diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp b/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.compnew file mode 100644index 0000000000..dfe329759c--- /dev/null+++ b/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp@@ -0,0 +1,12 @@+#version 450++#include "glu_head.glsl"++float op(float a, float b) {+ float gate = min(a, p.limit);+ float up = clamp(b, -p.limit, p.limit);++ return gate / (1.0f + exp(-gate)) * up;+}++#include "glu_main.glsl"diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cppindex d375c2d127..f0610f4cd8 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp@@ -986,6 +986,8 @@ void process_shaders() {string_to_spv("swiglu_f32", "swiglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});string_to_spv("swiglu_oai_f16", "swiglu_oai.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});string_to_spv("swiglu_oai_f32", "swiglu_oai.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});+ string_to_spv("swiglu_clamp_f16", "swiglu_clamp.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});+ string_to_spv("swiglu_clamp_f32", "swiglu_clamp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});string_to_spv("geglu_erf_f16", "geglu_erf.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});string_to_spv("geglu_erf_f32", "geglu_erf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});string_to_spv("geglu_quick_f16","geglu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hppindex 7a67ccf4fc..a7ff36030f 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp@@ -3101,6 +3101,10 @@ class ggml_webgpu_shader_lib {defines.push_back("OP_GEGLU_QUICK");variant += "_geglu_quick";break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ defines.push_back("OP_SWIGLU_CLAMP");+ variant += "_swiglu_clamp";+ break;default:GGML_ABORT("Unsupported GLU op");}diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cppindex 2434848a55..b953118a7a 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp@@ -2835,7 +2835,7 @@ static webgpu_encoded_op ggml_webgpu_glu(webgpu_context & ctx,(uint32_t) dst->ne[2],(uint32_t) ((int32_t *) dst->op_params)[1], // swappedggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 2)), // alpha, for swiglu_oai- ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 3)), // limit, for swiglu_oai+ ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 3)), // limit};std::vector<wgpu::BindGroupEntry> entries;@@ -4483,6 +4483,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, constcase GGML_GLU_OP_SWIGLU:case GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:+ case GGML_GLU_OP_SWIGLU_CLAMP:supports_op = op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;break;case GGML_GLU_OP_SWIGLU_OAI:diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgslindex d03f1c207d..6bbed5d3bf 100644--- a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl+++ b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl@@ -37,6 +37,14 @@ fn op(a: f32, b: f32) -> f32 {return out_glu;}#endif+#ifdef OP_SWIGLU_CLAMP+fn op(a: DataType, b: DataType) -> DataType {+ let limit = DataType(params.limit);+ let gate = min(a, limit);+ let up = clamp(b, -limit, limit);+ return gate / (1.0 + exp(-gate)) * up;+}+#endif#ifdef OP_GEGLU_ERFconst p_erf: DataType = 0.3275911;const a1_erf: DataType = 0.254829592;diff --git a/ggml/src/ggml.c b/ggml/src/ggml.cindex e0b615c07e..3bd3e3fe5e 100644--- a/ggml/src/ggml.c+++ b/ggml/src/ggml.c@@ -1253,10 +1253,10 @@ static const char * GGML_GLU_OP_NAME[GGML_GLU_OP_COUNT] = {"SWIGLU_OAI","GEGLU_ERF","GEGLU_QUICK",+ "SWIGLU_CLAMP",};-static_assert(GGML_GLU_OP_COUNT == 6, "GGML_GLU_OP_COUNT != 6");-+static_assert(GGML_GLU_OP_COUNT == 7, "GGML_GLU_OP_COUNT != 7");static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");static_assert(sizeof(struct ggml_tensor)%GGML_MEM_ALIGN == 0, "ggml_tensor size must be a multiple of GGML_MEM_ALIGN");@@ -3119,6 +3119,17 @@ struct ggml_tensor * ggml_swiglu_oai(return result;}+struct ggml_tensor * ggml_swiglu_clamp(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ float limit) {+ struct ggml_tensor * result = ggml_glu_impl(ctx, a, b, GGML_GLU_OP_SWIGLU_CLAMP, false);+ ggml_set_op_params_f32(result, 3, limit);++ return result;+}+// ggml_normstatic struct ggml_tensor * ggml_norm_impl(diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 8fca8e1bc0..72db486cac 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -1776,14 +1776,11 @@ ggml_tensor * llm_graph_context::build_ffn(const float limit = hparams.swiglu_clamp_shexp[il];constexpr float eps = 1e-6f;if (limit > eps) {- tmp = ggml_clamp(ctx0, tmp, -limit, limit);- cb(tmp, "ffn_up_clamped", il);-if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {- cur = ggml_clamp(ctx0, cur, -INFINITY, limit);- cb(cur, "ffn_gate_clamped", il);- cur = ggml_swiglu_split(ctx0, cur, tmp);+ cur = ggml_swiglu_clamp(ctx0, cur, tmp, limit);} else {+ tmp = ggml_clamp(ctx0, tmp, -limit, limit);+ cb(tmp, "ffn_up_clamped", il);ggml_tensor * gate_act = ggml_silu(ctx0, cur);cb(gate_act, "ffn_silu", il);gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);@@ -2173,14 +2170,11 @@ ggml_tensor * llm_graph_context::build_moe_ffn(const float limit = hparams.swiglu_clamp_exp[il];constexpr float eps = 1e-6f;if (limit > eps) {- up = ggml_clamp(ctx0, up, -limit, limit);- cb(up, "ffn_moe_up_clamped", il);-if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {- cur = ggml_clamp(ctx0, cur, -INFINITY, limit);- cb(cur, "ffn_moe_gate_clamped", il);- cur = ggml_swiglu_split(ctx0, cur, up);+ cur = ggml_swiglu_clamp(ctx0, cur, up, limit);} else {+ up = ggml_clamp(ctx0, up, -limit, limit);+ cb(up, "ffn_moe_up_clamped", il);ggml_tensor * gate_act = ggml_silu(ctx0, cur);cb(gate_act, "ffn_moe_silu", il);gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 4a7a062317..996b88db29 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -2243,6 +2243,63 @@ struct test_swiglu_oai : public test_case {}};+struct test_swiglu_clamp : public test_case {+ const ggml_type type;+ const std::array<int64_t, 4> ne_a;+ int v; // view (1 : non-contiguous a)+ float limit;++ std::string vars() override {+ return VARS_TO_STR4(type, ne_a, v, limit);+ }++ test_swiglu_clamp(ggml_type type = GGML_TYPE_F32,+ std::array<int64_t, 4> ne_a = {128, 2, 2, 2},+ int v = 0,+ float limit = 7.0f)+ : type(type), ne_a(ne_a), v(v), limit(limit) {}++ ggml_tensor * build_graph(ggml_context * ctx) override {+ ggml_tensor * a;+ ggml_tensor * b;+ if (v & 1) {+ auto ne = ne_a; ne[0] *= 3;+ a = ggml_new_tensor(ctx, type, 4, ne.data());+ ggml_set_param(a);+ ggml_set_name(a, "a");++ a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0);+ ggml_set_name(a, "view_of_a");++ b = ggml_new_tensor(ctx, type, 4, ne.data());+ ggml_set_param(b);+ ggml_set_name(b, "b");++ b = ggml_view_4d(ctx, b, ne_a[0], ne_a[1], ne_a[2], ne_a[3], b->nb[1], b->nb[2], b->nb[3], 0);+ ggml_set_name(b, "view_of_b");+ } else {+ a = ggml_new_tensor(ctx, type, 4, ne_a.data());+ ggml_set_param(a);+ ggml_set_name(a, "a");++ b = ggml_new_tensor(ctx, type, 4, ne_a.data());+ ggml_set_param(b);+ ggml_set_name(b, "b");+ }++ ggml_tensor * out = ggml_swiglu_clamp(ctx, a, b, limit);+ ggml_set_name(out, "out");++ return out;+ }++ void initialize_tensors(ggml_context * ctx) override {+ for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {+ init_tensor_uniform(t, -150.f, 150.f);+ }+ }+};+// GGML_OP_GET_ROWSstruct test_get_rows : public test_case {const ggml_type type;@@ -6380,6 +6437,9 @@ struct test_mul_mat_vec_fusion : public test_case {constexpr float alpha = 1.702f;constexpr float limit = 7.0f;out = ggml_swiglu_oai(ctx, ffn_gate, ffn_up, alpha, limit);+ } else if (glu_op == GGML_GLU_OP_SWIGLU_CLAMP) {+ constexpr float limit = 10.0f;+ out = ggml_swiglu_clamp(ctx, ffn_gate, ffn_up, limit);} else {out = ggml_glu_split(ctx, ffn_gate, ffn_up, glu_op);}@@ -8376,8 +8436,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {for (int v : {0, 1}) {for (int op = 0; op < GGML_GLU_OP_COUNT; op++) {- if (op == GGML_GLU_OP_SWIGLU_OAI) {- // SWIGLU_OAI is handled separately+ if (op == GGML_GLU_OP_SWIGLU_OAI || op == GGML_GLU_OP_SWIGLU_CLAMP) {continue;}@@ -8400,6 +8459,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}}+ for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {+ for (int v : {0, 1}) {+ for (float limit : {2.0f, 10.0f}) {+ test_cases.emplace_back(new test_swiglu_clamp(type, { 128, 2, 2, 2 }, v, limit));+ }+ }+ }+for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_Q4_0}) {test_cases.emplace_back(new test_get_rows(type, 300*256, 5, 4, 1, 2, false));test_cases.emplace_back(new test_get_rows(type, 256, 80000, 70000, 2, 1, false));@@ -10026,7 +10093,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {if (!with_gate && !with_bias) {continue;}- for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {+ for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_CLAMP}) {if (!with_bias && glu_op == GGML_GLU_OP_SWIGLU_OAI) {continue;}@@ -10041,7 +10108,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {use_id, 16, 8, b, with_bias, with_gate, with_lane_scale));test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}));- if (!use_id && with_gate && !with_bias) {+ if (!use_id && with_gate && !with_bias && glu_op != GGML_GLU_OP_SWIGLU_CLAMP) {// small multi-token batches (speculative decoding / MTP verify)for (int64_t m_batch : { 2, 4, 8 }) {test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, m_batch, 32, 256,@@ -10056,6 +10123,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}}+ for (bool b : {false, true}) {+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_IQ2_S, GGML_GLU_OP_SWIGLU_CLAMP, 1, 32, 256,+ true, 16, 8, b, false, true, false));+ }+for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) {for (bool with_norm : {false, true}) {for (bool bias_probs : {false, true}) {