Skip to content

LLM inference in C/C++

C++ ggml

ggml: add SWIGLU_CLAMP (#27930)

Aman Gupta0190529ec450
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 5f6774a630..26f31232f6 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -627,6 +627,7 @@ extern "C" {
GGML_GLU_OP_SWIGLU_OAI,
GGML_GLU_OP_GEGLU_ERF,
GGML_GLU_OP_GEGLU_QUICK,
+ GGML_GLU_OP_SWIGLU_CLAMP,
GGML_GLU_OP_COUNT,
};
@@ -1367,6 +1368,12 @@ extern "C" {
float alpha,
float limit);
+ GGML_API struct ggml_tensor * ggml_swiglu_clamp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float limit);
+
// normalize along rows
GGML_API struct ggml_tensor * ggml_norm(
struct ggml_context * ctx,
diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cpp
index 2dc0f40917..902d2eda69 100644
--- a/ggml/src/ggml-cann/aclnn_ops.cpp
+++ b/ggml/src/ggml-cann/aclnn_ops.cpp
@@ -211,6 +211,50 @@ void ggml_cann_swiglu(ggml_backend_cann_context & ctx, ggml_tensor * dst) {
GGML_CANN_CALL_ACLNN_OP(ctx, SwiGlu, acl_src.get(), (int64_t)2, acl_dst.get());
}
+void ggml_cann_swiglu_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {
+ ggml_tensor * src0 = dst->src[0];
+ ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
+
+ const int32_t swapped = ggml_get_op_params_i32(dst, 1);
+ acl_tensor_ptr acl_gate;
+ acl_tensor_ptr acl_up;
+ if (src1) {
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(src0->type == src1->type);
+ acl_gate = ggml_cann_create_tensor(src0);
+ acl_up = ggml_cann_create_tensor(src1);
+ } else {
+ int64_t ne[] = { src0->ne[0] / 2, src0->ne[1], src0->ne[2], src0->ne[3] };
+ size_t nb[] = { src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3] };
+ acl_gate = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, 0);
+ acl_up = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, ne[0] * ggml_element_size(src0));
+ if (swapped) {
+ std::swap(acl_gate, acl_up);
+ }
+ }
+
+ ggml_cann_pool_alloc temp_alloc(ctx.pool(), ggml_nbytes(dst));
+ acl_tensor_ptr acl_temp = ggml_cann_create_tensor(temp_alloc.get(), ggml_cann_type_mapping(dst->type),
+ ggml_element_size(dst), dst->ne, dst->nb, GGML_MAX_DIMS);
+ acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);
+
+ const float limit = ggml_get_op_params_f32(dst, 3);
+ float min_gate = -INFINITY;
+ float min_up = -limit;
+ float max_value = limit;
+ acl_scalar_ptr acl_min_gate = ggml_cann_create_scalar(&min_gate, ACL_FLOAT);
+ acl_scalar_ptr acl_min_up = ggml_cann_create_scalar(&min_up, ACL_FLOAT);
+ acl_scalar_ptr acl_limit = ggml_cann_create_scalar(&max_value, ACL_FLOAT);
+
+ GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_gate.get(), acl_min_gate.get(), acl_limit.get(), acl_temp.get());
+ GGML_CANN_CALL_ACLNN_OP(ctx, Silu, acl_temp.get(), acl_dst.get());
+ GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_up.get(), acl_min_up.get(), acl_limit.get(), acl_temp.get());
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_dst.get(), acl_temp.get());
+}
+
// Fused GeGLU using aclnnGeGluV3: splits input along ne[0] (CANN last dim),
// activates the LEFT half with GELU, multiplies by right half.
// approximate: 0=tanh, 1=none(erf). activateLeft=true matches GGML convention.
@@ -4433,4 +4477,3 @@ void ggml_cann_gated_linear_attn(ggml_backend_cann_context & ctx, ggml_tensor *
}
}
}
-
diff --git a/ggml/src/ggml-cann/aclnn_ops.h b/ggml/src/ggml-cann/aclnn_ops.h
index cdbf9260f8..678f4d654e 100644
--- a/ggml/src/ggml-cann/aclnn_ops.h
+++ b/ggml/src/ggml-cann/aclnn_ops.h
@@ -76,6 +76,7 @@
void ggml_cann_repeat(ggml_backend_cann_context & ctx, ggml_tensor * dst);
void ggml_cann_swiglu(ggml_backend_cann_context & ctx, ggml_tensor * dst);
+void ggml_cann_swiglu_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst);
void ggml_cann_geglu(ggml_backend_cann_context & ctx, ggml_tensor * dst, int64_t approximate);
/**
diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp
index 5e5541aac9..c2745014a1 100644
--- a/ggml/src/ggml-cann/ggml-cann.cpp
+++ b/ggml/src/ggml-cann/ggml-cann.cpp
@@ -1872,6 +1872,9 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg
case GGML_GLU_OP_SWIGLU:
ggml_cann_swiglu(ctx, dst);
break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ ggml_cann_swiglu_clamp(ctx, dst);
+ break;
case GGML_GLU_OP_GEGLU_QUICK:
ggml_cann_geglu_quick(ctx, dst);
break;
@@ -2428,6 +2431,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
case GGML_GLU_OP_SWIGLU:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
return true;
default:
return false;
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index b9c0fa3ddc..6bc4467e37 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -2311,6 +2311,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
{
n_tasks = n_threads;
} break;
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index b47ce5463c..266261c5e5 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -3403,6 +3403,139 @@ static void ggml_compute_forward_swiglu_oai(
}
}
+// ggml_compute_forward_swiglu_clamp
+
+static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * params, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ char * src0_d = (char *) src0->data;
+ char * src1_d = (char *) (src1 ? src1->data : src0->data);
+ const size_t src0_o = src0->nb[1];
+ const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
+
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
+
+ if (src1) {
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(src0->type == src1->type);
+ }
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
+ const int nr = ggml_nrows(src0);
+
+ GGML_ASSERT(dst->ne[0] == nc);
+ GGML_ASSERT(ggml_nrows(dst) == nr);
+
+ const int32_t swapped = ggml_get_op_params_i32(dst, 1);
+ const float limit = ggml_get_op_params_f32(dst, 3);
+
+ const int dr = (nr + nth - 1) / nth;
+ const int ir0 = dr * ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ float * src0_p = (float *) (src0_d + i1 * src0_o);
+ float * src1_p = (float *) (src1_d + i1 * src1_o);
+ float * dst_p = (float *) ((char *) dst->data + i1 * (dst->nb[1]));
+
+ if (!src1) {
+ src0_p += swapped ? nc : 0;
+ src1_p += swapped ? 0 : nc;
+ }
+
+ for (int k = 0; k < nc; k++) {
+ const float gate = std::min(src0_p[k], limit);
+ const float up = std::clamp(src1_p[k], -limit, limit);
+ dst_p[k] = gate / (1.f + expf(-gate)) * up;
+ }
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = dst_p[k];
+ GGML_UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif // NDEBUG
+ }
+}
+
+static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ char * src0_d = (char *) src0->data;
+ char * src1_d = (char *) (src1 ? src1->data : src0->data);
+ const size_t src0_o = src0->nb[1];
+ const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
+
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
+
+ if (src1) {
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(src0->type == src1->type);
+ }
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
+ const int nr = ggml_nrows(src0);
+
+ GGML_ASSERT(dst->ne[0] == nc);
+ GGML_ASSERT(ggml_nrows(dst) == nr);
+
+ const int32_t swapped = ggml_get_op_params_i32(dst, 1);
+ const float limit = ggml_get_op_params_f32(dst, 3);
+
+ const int dr = (nr + nth - 1) / nth;
+ const int ir0 = dr * ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o);
+ ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o);
+ ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1]));
+
+ if (!src1) {
+ src0_p += swapped ? nc : 0;
+ src1_p += swapped ? 0 : nc;
+ }
+
+ for (int k = 0; k < nc; k++) {
+ const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit);
+ const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit);
+ dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up);
+ }
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = GGML_FP16_TO_FP32(dst_p[k]);
+ GGML_UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif // NDEBUG
+ }
+}
+
+static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params, ggml_tensor * dst) {
+ switch (dst->src[0]->type) {
+ case GGML_TYPE_F32:
+ ggml_compute_forward_swiglu_clamp_f32(params, dst);
+ break;
+ case GGML_TYPE_F16:
+ ggml_compute_forward_swiglu_clamp_f16(params, dst);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
// ggml_compute_forward_geglu_erf
static void ggml_compute_forward_geglu_erf_f32(
@@ -10136,6 +10269,10 @@ void ggml_compute_forward_glu(
{
ggml_compute_forward_geglu_quick(params, dst);
} break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ {
+ ggml_compute_forward_swiglu_clamp(params, dst);
+ } break;
default:
{
GGML_ABORT("fatal error");
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
index 14dd1098c9..e5ccd1feab 100644
--- a/ggml/src/ggml-cuda/common.cuh
+++ b/ggml/src/ggml-cuda/common.cuh
@@ -1539,6 +1539,7 @@ struct ggml_cuda_mm_fusion_args_host {
const ggml_tensor * x_scale = nullptr;
const ggml_tensor * gate_scale = nullptr;
ggml_glu_op glu_op;
+ float glu_limit = 0.0f;
};
struct ggml_cuda_mm_fusion_args_device {
const void * x_bias = nullptr;
@@ -1547,6 +1548,7 @@ struct ggml_cuda_mm_fusion_args_device {
const void * x_scale = nullptr;
const void * gate_scale = nullptr;
ggml_glu_op glu_op;
+ float glu_limit = 0.0f;
};
struct ggml_cuda_kernel_launch_params {
@@ -1673,4 +1675,3 @@ static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_ke
kernel<<<launch_params.block_nums, launch_params.block_dims, launch_params.shmem, launch_params.stream>>>(std::forward<Args>(args)... );
CUDA_CHECK(cudaGetLastError());
}
-
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index bd9754c2ff..7c2d3cad20 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -1744,7 +1744,7 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up,
return false;
}
- static constexpr std::array<ggml_glu_op, 3> valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI };
+ static constexpr std::array<ggml_glu_op, 4> valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI, GGML_GLU_OP_SWIGLU_CLAMP };
if (std::find(valid_glu_ops.begin(), valid_glu_ops.end(), ggml_get_glu_op(glu)) == valid_glu_ops.end()) {
return false;
@@ -2203,6 +2203,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
case GGML_GLU_OP_GEGLU_QUICK:
ggml_cuda_op_geglu_quick(ctx, dst);
break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ ggml_cuda_op_swiglu_clamp(ctx, dst);
+ break;
default:
return false;
}
@@ -3595,6 +3598,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
fusion_data.x_scale = up_scale;
fusion_data.gate_scale = gate_scale;
fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);
@@ -3688,6 +3692,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
fusion_data.x_scale = up_scale;
fusion_data.gate_scale = gate_scale;
fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);
@@ -3744,6 +3749,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
fusion_data.x_bias = up_bias_tensor;
fusion_data.gate_bias = gate_bias_tensor;
fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data);
fused_mul_mat_vec = true;
@@ -3757,6 +3763,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
fusion_data.x_bias = up_bias_tensor;
fusion_data.gate_bias = gate_bias_tensor;
fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data);
fused_mul_mat_vec = true;
@@ -3781,8 +3788,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) {
ggml_cuda_mm_fusion_args_host fusion_data{};
- fusion_data.gate = gate->src[0];
- fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.gate = gate->src[0];
+ fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data);
fused_mul_mat_vec = true;
@@ -3792,8 +3800,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
if (ggml_cuda_should_fuse_mul_mat_vec_q(up)) {
ggml_cuda_mm_fusion_args_host fusion_data{};
- fusion_data.gate = gate->src[0];
- fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.gate = gate->src[0];
+ fusion_data.glu_op = ggml_get_glu_op(glu);
+ fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data);
fused_mul_mat_vec = true;
@@ -4919,6 +4928,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
return ggml_is_contiguous_1(op->src[0]);
default:
return false;
diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index d7dbc8b992..bd5c5d421a 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -56,6 +56,7 @@ static __global__ void mul_mat_vec_f(
bool use_bias = false;
bool use_gate_bias = false;
ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU;
+ float glu_limit = 0.0f;
const T * gate_x = nullptr;
const float * x_bias = nullptr;
const float * gate_bias = nullptr;
@@ -65,6 +66,7 @@ static __global__ void mul_mat_vec_f(
use_bias = fusion.x_bias != nullptr;
use_gate_bias = fusion.gate_bias != nullptr;
glu_op = fusion.glu_op;
+ glu_limit = fusion.glu_limit;
if (use_gate) {
gate_x = static_cast<const T *>(fusion.gate);
@@ -365,6 +367,9 @@ static __global__ void mul_mat_vec_f(
value = ggml_cuda_op_swiglu_oai_single(gate_value, value);
break;
}
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ value = ggml_cuda_op_swiglu_clamp_single(gate_value, value, glu_limit);
+ break;
default:
break;
}
@@ -374,7 +379,7 @@ static __global__ void mul_mat_vec_f(
dst[tid*stride_col_dst + row] = value;
if constexpr (!has_fusion) {
- GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, gate_x, x_bias, gate_bias, sumf_gate);
+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, glu_limit, gate_x, x_bias, gate_bias, sumf_gate);
}
}
@@ -675,6 +680,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
fusion_local.gate_bias = fusion->gate_bias->data;
}
fusion_local.glu_op = fusion->glu_op;
+ fusion_local.glu_limit = fusion->glu_limit;
}
const int64_t s01 = src0->nb[1] / ts_src0;
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index 9705348098..79f7a3f6fe 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -595,6 +595,7 @@ static __global__ void mul_mat_vec_q(
const float * x_scale = nullptr;
const float * gate_scale = nullptr;
ggml_glu_op active_glu;
+ float glu_limit = 0.0f;
if constexpr (has_fusion) {
use_gate = fusion.gate != nullptr;
@@ -604,6 +605,7 @@ static __global__ void mul_mat_vec_q(
x_bias = (const float *) fusion.x_bias;
gate_bias = (const float *) fusion.gate_bias;
active_glu = fusion.glu_op;
+ glu_limit = fusion.glu_limit;
if constexpr (type == GGML_TYPE_NVFP4) {
use_scale = fusion.x_scale != nullptr;
use_gate_scale = fusion.gate_scale != nullptr && use_gate;
@@ -745,6 +747,9 @@ static __global__ void mul_mat_vec_q(
case GGML_GLU_OP_SWIGLU_OAI:
result = ggml_cuda_op_swiglu_oai_single(gate_value, result);
break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);
+ break;
default:
result = result * gate_value;
break;
@@ -757,7 +762,7 @@ static __global__ void mul_mat_vec_q(
}
if constexpr (!has_fusion) {
- GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);
+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, glu_limit, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);
}
if constexpr (type != GGML_TYPE_NVFP4) {
GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales);
@@ -1310,6 +1315,7 @@ void ggml_cuda_mul_mat_vec_q(
fusion_local.gate_scale = fusion->gate_scale->data;
}
fusion_local.glu_op = fusion->glu_op;
+ fusion_local.glu_limit = fusion->glu_limit;
}
// If src0 is a temporary compute buffer, clear any potential padding.
diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu
index 4cb805fa60..d3e594878f 100644
--- a/ggml/src/ggml-cuda/unary.cu
+++ b/ggml/src/ggml-cuda/unary.cu
@@ -427,6 +427,81 @@ void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
swiglu_oai_cuda(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream);
}
+// swiglu_clamp
+
+template <typename T>
+static __global__ void swiglu_clamp_kernel(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, float limit) {
+ const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x;
+
+ if (i >= k) {
+ return;
+ }
+
+ const int64_t j0 = (i / n) * o0 + (i % n);
+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
+
+ dst[i] = (T) ggml_cuda_op_swiglu_clamp_single((float) gate[j0], (float) up[j1], limit);
+}
+
+template <typename T>
+static void swiglu_clamp_cuda(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, const float limit, cudaStream_t stream) {
+ const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE;
+ swiglu_clamp_kernel<<<num_blocks, CUDA_GLU_BLOCK_SIZE, 0, stream>>>(gate, up, dst, k, n, o0, o1, limit);
+}
+
+void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ void * src0_d = src0->data;
+ void * src1_d = src1 ? src1->data : src0->data;
+ const int64_t src0_o = src0->nb[1];
+ const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
+ void * dst_d = dst->data;
+ const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+ GGML_ASSERT(src0->nb[0] == ggml_element_size(src0));
+ GGML_ASSERT(ggml_is_contiguous(dst));
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == dst->type);
+ GGML_ASSERT(dst->ne[0] == nc);
+ GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0));
+
+ if (src1) {
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));
+ GGML_ASSERT(src1->ne[0] == nc);
+ GGML_ASSERT(src0->type == src1->type);
+ }
+
+ const int32_t swapped = ggml_get_op_params_i32(dst, 1);
+ const float limit = ggml_get_op_params_f32(dst, 3);
+
+ if (src0->type == GGML_TYPE_F16) {
+ half * src0_p = (half *) src0_d;
+ half * src1_p = (half *) src1_d;
+
+ if (!src1) {
+ src0_p += swapped ? nc : 0;
+ src1_p += swapped ? 0 : nc;
+ }
+
+ swiglu_clamp_cuda(src0_p, src1_p, (half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(half), src1_o / sizeof(half), limit, stream);
+ } else {
+ float * src0_p = (float *) src0_d;
+ float * src1_p = (float *) src1_d;
+
+ if (!src1) {
+ src0_p += swapped ? nc : 0;
+ src1_p += swapped ? 0 : nc;
+ }
+
+ swiglu_clamp_cuda(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), limit, stream);
+ }
+}
+
/* CUDA kernel + launcher for xIELU */
template <typename T>
diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuh
index 81ed873ecc..04f3af6443 100644
--- a/ggml/src/ggml-cuda/unary.cuh
+++ b/ggml/src/ggml-cuda/unary.cuh
@@ -83,6 +83,8 @@ void ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
void ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
void ggml_cuda_op_geglu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
@@ -112,3 +114,10 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_oai_single(float x, float g
out_glu = out_glu * (1.0f + g);
return out_glu;
}
+
+__device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, float up, float limit) {
+ gate = fminf(gate, limit);
+ up = fmaxf(fminf(up, limit), -limit);
+
+ return ggml_cuda_op_silu_single(gate) * up;
+}
diff --git a/ggml/src/ggml-et/et-kernels/src/glu_f32.c b/ggml/src/ggml-et/et-kernels/src/glu_f32.c
index 95fe572158..d376d6f56f 100644
--- a/ggml/src/ggml-et/et-kernels/src/glu_f32.c
+++ b/ggml/src/ggml-et/et-kernels/src/glu_f32.c
@@ -17,7 +17,7 @@ struct ggml_et_glu_params {
int32_t glu_op_type; // GLU operation type (REGLU=0, GEGLU=1, SWIGLU=2, etc.)
int32_t swapped; // Whether gate and value are swapped
float alpha; // SWIGLU_OAI: sigmoid scaling factor
- float limit; // SWIGLU_OAI: clamp limit
+ float limit; // GLU clamp limit
};
// SiLU activation function: silu(x) = x * sigmoid(x) = x / (1 + exp(-x))
@@ -332,6 +332,57 @@ static inline void block_swiglu_oai(float * dst_block,
}
}
+static inline void block_swiglu_clamp(float * dst_block,
+ const float * gate_block,
+ const float * up_block,
+ int elements,
+ float limit) {
+ int32_t vec_end = (elements / 8) * 8;
+
+ unsigned long temp_mask;
+ __asm__ volatile("mova.x.m %0" : "=r"(temp_mask));
+ __asm__ volatile("mov.m.x m0, x0, 0xFF");
+
+ float one_const = 1.0f;
+ float limit_pos = limit;
+ float limit_neg = -limit;
+ float neg_log2e = -1.4426950408889634f;
+
+ for (int32_t i = 0; i < vec_end; i += 8) {
+ __asm__ volatile(
+ "flw.ps f10, %[gate_vec]\n"
+ "flw.ps f11, %[up_vec]\n"
+ "fbc.ps f21, %[one_ptr]\n"
+ "fbc.ps f23, %[lim_pos]\n"
+ "fbc.ps f24, %[lim_neg]\n"
+ "fbc.ps f25, %[k_ptr]\n"
+ "fmin.ps f12, f10, f23\n"
+ "fmax.ps f13, f11, f24\n"
+ "fmin.ps f13, f13, f23\n"
+ "fmul.ps f14, f12, f25\n"
+ "fexp.ps f15, f14\n"
+ "fadd.ps f15, f15, f21\n"
+ "frcp.ps f16, f15\n"
+ "fmul.ps f17, f12, f16\n"
+ "fmul.ps f18, f17, f13\n"
+ "fsw.ps f18, %[dst_out]\n"
+ : [dst_out] "=m"(*(float (*)[8]) & dst_block[i])
+ : [gate_vec] "m"(*(const float (*)[8]) & gate_block[i]), [up_vec] "m"(*(const float (*)[8]) & up_block[i]),
+ [one_ptr] "m"(one_const), [lim_pos] "m"(limit_pos), [lim_neg] "m"(limit_neg), [k_ptr] "m"(neg_log2e)
+ : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f21", "f23", "f24", "f25");
+ }
+
+ __asm__ volatile("mova.m.x %0" :: "r"(temp_mask));
+
+ for (int32_t i = vec_end; i < elements; i++) {
+ float gate = gate_block[i] > limit ? limit : gate_block[i];
+ float up = up_block[i];
+ up = up > limit ? limit : up;
+ up = up < -limit ? -limit : up;
+ dst_block[i] = silu_f32(gate) * up;
+ }
+}
+
// Scalar erf approximation (Abramowitz & Stegun 7.1.26, max error ~1.5e-7)
static inline float erf_approx(float x) {
const float a1 = 0.254829592f;
@@ -386,6 +437,7 @@ int entry_point(struct ggml_et_glu_params * params, void * env) {
switch (params->glu_op_type) {
case GGML_GLU_OP_SWIGLU:
case GGML_GLU_OP_SWIGLU_OAI:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
case GGML_GLU_OP_GEGLU:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
@@ -531,6 +583,9 @@ int entry_point(struct ggml_et_glu_params * params, void * env) {
case GGML_GLU_OP_SWIGLU_OAI:
block_swiglu_oai(dst_ptr, x_ptr, g_ptr, (int) elements_to_process, params->alpha, params->limit);
break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ block_swiglu_clamp(dst_ptr, x_ptr, g_ptr, (int) elements_to_process, params->limit);
+ break;
default:
return -1;
}
diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp
index b37f6d261d..5771679b3f 100644
--- a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp
+++ b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp
@@ -261,7 +261,12 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ct
GGML_LOG_ERROR("ET: GLU CPU comparison requires split tensor mode\n");
return false;
}
- ctx->cpu_dst = ggml_glu_split(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, glu_op);
+ if (glu_op == GGML_GLU_OP_SWIGLU_CLAMP) {
+ const float limit = ggml_get_op_params_f32(node, 3);
+ ctx->cpu_dst = ggml_swiglu_clamp(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, limit);
+ } else {
+ ctx->cpu_dst = ggml_glu_split(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, glu_op);
+ }
}
break;
case GGML_OP_SOFT_MAX:
diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp
index 7871d52408..8765138672 100644
--- a/ggml/src/ggml-et/ggml-et-ops.cpp
+++ b/ggml/src/ggml-et/ggml-et-ops.cpp
@@ -636,6 +636,7 @@ bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensor
case GGML_GLU_OP_GEGLU:
case GGML_GLU_OP_SWIGLU:
case GGML_GLU_OP_SWIGLU_OAI:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
break;
@@ -661,6 +662,8 @@ bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensor
params.limit = 0.0f;
if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI) {
params.alpha = ggml_get_op_params_f32(node, 2);
+ }
+ if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI || glu_op_type == GGML_GLU_OP_SWIGLU_CLAMP) {
params.limit = ggml_get_op_params_f32(node, 3);
}
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp
index b87b189a57..61c31d6f29 100644
--- a/ggml/src/ggml-et/ggml-et.cpp
+++ b/ggml/src/ggml-et/ggml-et.cpp
@@ -1210,7 +1210,8 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm
// Check GLU variant - support SWIGLU, SWIGLU_OAI, GEGLU, GEGLU_ERF, GEGLU_QUICK, REGLU
ggml_glu_op glu_type = ggml_get_glu_op(op);
const bool supported_variant = glu_type == GGML_GLU_OP_SWIGLU || glu_type == GGML_GLU_OP_SWIGLU_OAI ||
- glu_type == GGML_GLU_OP_GEGLU || glu_type == GGML_GLU_OP_GEGLU_ERF ||
+ glu_type == GGML_GLU_OP_SWIGLU_CLAMP || glu_type == GGML_GLU_OP_GEGLU ||
+ glu_type == GGML_GLU_OP_GEGLU_ERF ||
glu_type == GGML_GLU_OP_GEGLU_QUICK || glu_type == GGML_GLU_OP_REGLU;
if (op->src[1]) {
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 87e6989bd2..3eb84fd2a9 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -4701,6 +4701,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
switch (ggml_get_glu_op(t)) {
case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;
case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;
+ case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;
case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;
default: break;
}
@@ -5528,6 +5529,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
switch (ggml_get_glu_op(op)) {
case GGML_GLU_OP_SWIGLU:
case GGML_GLU_OP_SWIGLU_OAI:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
case GGML_GLU_OP_GEGLU:
supp = ggml_hexagon_supported_activations(sess, op);
break;
diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c
index 0a8bf84e38..ac00b447d9 100644
--- a/ggml/src/ggml-hexagon/htp/act-ops.c
+++ b/ggml/src/ggml-hexagon/htp/act-ops.c
@@ -180,6 +180,26 @@ static void swiglu_oai_f32(const float * restrict src0,
}
}
+static void swiglu_clamp_f32(const float * restrict src0,
+ const float * restrict src1,
+ float * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_act_context * actx) {
+ htp_glu_op_preamble;
+ const float limit = ((const float *) (actx->octx->op_params))[3];
+
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
+
+ hvx_min_scalar_f32((uint8_t *) src0_ptr, src0_ptr, limit, nc);
+ hvx_clamp_scalar_f32((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);
+ hvx_sigmoid_f32_aa(dst_ptr, src0_ptr, nc);
+ hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);
+ }
+}
+
static const float GELU_COEF_A = 0.044715f;
static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
@@ -411,6 +431,7 @@ static void geglu_f32(const float * restrict src0,
DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx))
DEFINE_GLU_PER_THREAD(swiglu_oai, "swiglu-oai-f32", swiglu_oai_f32(src0_spad, src1_spad, dst_spad, block_size, actx))
+DEFINE_GLU_PER_THREAD(swiglu_clamp, "swiglu-clamp-f32", swiglu_clamp_f32(src0_spad, src1_spad, dst_spad, block_size, actx))
DEFINE_GLU_PER_THREAD(geglu, "geglu-f32", geglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx))
static int execute_op_activations_f32(struct htp_ops_context * octx) {
@@ -437,6 +458,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
op_type = "swiglu-oai-f32";
break;
+ case HTP_OP_GLU_SWIGLU_CLAMP:
+ act_op_func = (worker_callback_t) glu_swiglu_clamp_f32_per_thread;
+ op_type = "swiglu-clamp-f32";
+ break;
+
case HTP_OP_GLU_GEGLU:
act_op_func = (worker_callback_t)glu_geglu_f32_per_thread;
op_type = "geglu-f32";
@@ -527,7 +553,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
const uint8_t * data_src0 = (const uint8_t *) src0->data;
const uint8_t * data_src1 = src1 ? (const uint8_t *) src1->data : NULL;
- if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_GEGLU)) {
+ if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_SWIGLU_CLAMP || octx->op == HTP_OP_GLU_GEGLU)) {
const int32_t swapped = octx->op_params[1];
data_src1 = data_src0;
actx.src1_row_size = actx.src0_row_size;
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index e804844d59..53c95f28d3 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -96,6 +96,7 @@ enum htp_op_code {
HTP_OP_FENCE,
HTP_OP_ALLREDUCE,
HTP_OP_ALLREDUCE_ADD,
+ HTP_OP_GLU_SWIGLU_CLAMP,
HTP_OP_INVALID
};
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index fe7d093a81..27d1dedcdf 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -784,6 +784,7 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_GLU_SWIGLU:
case HTP_OP_GLU_SWIGLU_OAI:
+ case HTP_OP_GLU_SWIGLU_CLAMP:
case HTP_OP_GLU_GEGLU:
return op_activations(octx);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index 4e855be446..5a2f01f1db 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -318,6 +318,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu(ggml_metal_l
case GGML_GLU_OP_SWIGLU_OAI: op_str = "swiglu_oai"; break;
case GGML_GLU_OP_GEGLU_ERF: op_str = "geglu_erf"; break;
case GGML_GLU_OP_GEGLU_QUICK: op_str = "geglu_quick"; break;
+ case GGML_GLU_OP_SWIGLU_CLAMP: op_str = "swiglu_clamp"; break;
default: GGML_ABORT("fatal error");
} break;
default: GGML_ABORT("fatal error");
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index a053887a33..83344539e5 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -1510,6 +1510,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
return ggml_is_contiguous_1(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
default:
return false;
diff --git a/ggml/src/ggml-metal/kernels/unary.metal b/ggml/src/ggml-metal/kernels/unary.metal
index 39cad0cbee..e50a648639 100644
--- a/ggml/src/ggml-metal/kernels/unary.metal
+++ b/ggml/src/ggml-metal/kernels/unary.metal
@@ -317,6 +317,32 @@ typedef decltype(kernel_swiglu_oai<float>) kernel_swiglu_oai_t;
template [[host_name("kernel_swiglu_oai_f32")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai<float>;
template [[host_name("kernel_swiglu_oai_f16")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai<half>;
+template<typename T>
+kernel void kernel_swiglu_clamp(
+ constant ggml_metal_kargs_glu & args,
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ uint tgpig[[threadgroup_position_in_grid]],
+ uint tpitg[[thread_position_in_threadgroup]],
+ uint ntg[[threads_per_threadgroup]]) {
+ device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
+ device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
+ device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
+
+ for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
+ const float gate = min((float) src0_row[i0], args.limit);
+ const float up = clamp((float) src1_row[i0], -args.limit, args.limit);
+
+ dst_row[i0] = (T)(gate / (1.0f + exp(-gate)) * up);
+ }
+}
+
+typedef decltype(kernel_swiglu_clamp<float>) kernel_swiglu_clamp_t;
+
+template [[host_name("kernel_swiglu_clamp_f32")]] kernel kernel_swiglu_clamp_t kernel_swiglu_clamp<float>;
+template [[host_name("kernel_swiglu_clamp_f16")]] kernel kernel_swiglu_clamp_t kernel_swiglu_clamp<half>;
+
template<typename T>
kernel void kernel_geglu_erf(
constant ggml_metal_kargs_glu & args,
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 426aac5231..90635cc858 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -744,8 +744,9 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_tri;
cl_kernel kernel_fill;
cl_kernel kernel_clamp;
- cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_geglu_erf, kernel_geglu_quick,
- kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_geglu_erf_f16, kernel_geglu_quick_f16;
+ cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf,
+ kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16,
+ kernel_geglu_erf_f16, kernel_geglu_quick_f16;
cl_kernel kernel_norm, kernel_norm_mul_add;
cl_kernel kernel_rms_norm, kernel_rms_norm_mul;
cl_kernel kernel_l2_norm_f32;
@@ -1601,11 +1602,13 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_reglu = clCreateKernel(backend_ctx->program_glu, "kernel_reglu", &err), err));
CL_CHECK((backend_ctx->kernel_swiglu = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu", &err), err));
CL_CHECK((backend_ctx->kernel_swiglu_oai = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_oai", &err), err));
+ CL_CHECK((backend_ctx->kernel_swiglu_clamp = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp", &err), err));
CL_CHECK((backend_ctx->kernel_geglu_erf = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf", &err), err));
CL_CHECK((backend_ctx->kernel_geglu_quick = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick", &err), err));
CL_CHECK((backend_ctx->kernel_geglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_f16", &err), err));
CL_CHECK((backend_ctx->kernel_reglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_reglu_f16", &err), err));
CL_CHECK((backend_ctx->kernel_swiglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_f16", &err), err));
+ CL_CHECK((backend_ctx->kernel_swiglu_clamp_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp_f16", &err), err));
CL_CHECK((backend_ctx->kernel_geglu_erf_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf_f16", &err), err));
CL_CHECK((backend_ctx->kernel_geglu_quick_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick_f16", &err), err));
GGML_LOG_CONT(".");
@@ -7700,6 +7703,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
default:
return false;
@@ -24886,6 +24890,13 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const
case GGML_GLU_OP_SWIGLU_OAI:
kernel = backend_ctx->kernel_swiglu_oai;
break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ if (dst->type == GGML_TYPE_F32) {
+ kernel = backend_ctx->kernel_swiglu_clamp;
+ } else {
+ kernel = backend_ctx->kernel_swiglu_clamp_f16;
+ }
+ break;
case GGML_GLU_OP_GEGLU_ERF:
if (dst->type == GGML_TYPE_F32) {
kernel = backend_ctx->kernel_geglu_erf;
@@ -24941,8 +24952,10 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00_off));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10_off));
- if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) {
+ if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_CLAMP) {
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &limit));
+ }
+ if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) {
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &alpha));
}
diff --git a/ggml/src/ggml-opencl/kernels/glu.cl b/ggml/src/ggml-opencl/kernels/glu.cl
index 059a4bbf1b..30bad00f7d 100644
--- a/ggml/src/ggml-opencl/kernels/glu.cl
+++ b/ggml/src/ggml-opencl/kernels/glu.cl
@@ -243,6 +243,71 @@ kernel void kernel_swiglu_oai(
}
}
+//------------------------------------------------------------------------------
+// swiglu_clamp
+//------------------------------------------------------------------------------
+kernel void kernel_swiglu_clamp(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ ulong nb01,
+ ulong nb11,
+ int ne0,
+ ulong nb1,
+ int ne00_off,
+ int ne10_off,
+ float limit
+) {
+ src0 = (global char*)((global char*)src0 + offset0);
+ src1 = (global char*)((global char*)src1 + offset1);
+ dst = (global char*)((global char*)dst + offsetd);
+
+ global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;
+ global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;
+ global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);
+
+ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
+ const float gate = min(src0_row[i0], limit);
+ const float up = clamp(src1_row[i0], -limit, limit);
+
+ dst_row[i0] = gate / (1.0f + exp(-gate)) * up;
+ }
+}
+
+kernel void kernel_swiglu_clamp_f16(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ ulong nb01,
+ ulong nb11,
+ int ne0,
+ ulong nb1,
+ int ne00_off,
+ int ne10_off,
+ float limit
+) {
+ src0 = (global char*)((global char*)src0 + offset0);
+ src1 = (global char*)((global char*)src1 + offset1);
+ dst = (global char*)((global char*)dst + offsetd);
+
+ global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;
+ global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;
+ global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1);
+
+ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
+ const float gate = min((float) src0_row[i0], limit);
+ const float up = clamp((float) src1_row[i0], -limit, limit);
+
+ dst_row[i0] = (half) (gate / (1.0f + exp(-gate)) * up);
+ }
+}
+
//------------------------------------------------------------------------------
// geglu_erf
//------------------------------------------------------------------------------
diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp
index d220f2f584..d81fc53b5d 100644
--- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp
@@ -89,6 +89,21 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
return rename_outputs_with_suffix({res}, context.get_name());
}
+OutputVector translate_glu_swiglu_clamp(const NodeContext & context) {
+ auto [src0, src1] = get_glu_inputs(context);
+
+ const int32_t * params = context.get_output_op_params();
+ const float limit = reinterpret_cast<const float *>(params)[3];
+
+ auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);
+ auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(gate);
+ auto silu = std::make_shared<ov::op::v1::Multiply>(gate, sigmoid);
+ auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);
+ auto res = std::make_shared<ov::op::v1::Multiply>(silu, up);
+
+ return rename_outputs_with_suffix({res}, context.get_name());
+}
+
} // namespace op
} // namespace ggml
} // namespace frontend
diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp
index 9c9d8eeac7..d4f5ac3073 100644
--- a/ggml/src/ggml-openvino/openvino/op_table.cpp
+++ b/ggml/src/ggml-openvino/openvino/op_table.cpp
@@ -60,6 +60,7 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
{"GGML_OP_VIEW", op::translate_view },
{"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu },
{"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai },
+ {"GGML_GLU_OP_SWIGLU_CLAMP", op::translate_glu_swiglu_clamp },
{"GGML_GLU_OP_GEGLU", op::translate_glu_geglu },
{"GGML_GLU_OP_GEGLU_QUICK", op::translate_glu_geglu_quick },
{"GGML_OP_SET_ROWS", op::translate_set_rows },
diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h
index 0a81a57a66..a0a42bff33 100644
--- a/ggml/src/ggml-openvino/openvino/op_table.h
+++ b/ggml/src/ggml-openvino/openvino/op_table.h
@@ -37,6 +37,7 @@ GGML_OP_CONVERTER(translate_transpose);
GGML_OP_CONVERTER(translate_view);
GGML_OP_CONVERTER(translate_glu_swiglu);
GGML_OP_CONVERTER(translate_glu_swiglu_oai);
+GGML_OP_CONVERTER(translate_glu_swiglu_clamp);
GGML_OP_CONVERTER(translate_glu_geglu);
GGML_OP_CONVERTER(translate_glu_geglu_quick);
GGML_OP_CONVERTER(translate_set_rows);
diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp
index 95914873e5..2e926abea7 100644
--- a/ggml/src/ggml-sycl/element_wise.cpp
+++ b/ggml/src/ggml-sycl/element_wise.cpp
@@ -1132,6 +1132,102 @@ void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst)
swiglu_oai_sycl(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream);
}
+template <typename T>
+static void swiglu_clamp_kernel(const T * gate,
+ const T * up,
+ T * dst,
+ const int64_t k,
+ const int64_t n,
+ const int64_t o0,
+ const int64_t o1,
+ float limit,
+ sycl::nd_item<3> item_ct1) {
+ const int64_t i = int64_t(item_ct1.get_local_range(2)) * item_ct1.get_group(2) + item_ct1.get_local_id(2);
+
+ if (i >= k) {
+ return;
+ }
+
+ const int64_t j0 = (i / n) * o0 + (i % n);
+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
+
+ const float gate_value = sycl::fmin((float) gate[j0], limit);
+ const float up_value = sycl::fmax(sycl::fmin((float) up[j1], limit), -limit);
+ dst[i] = (T) (gate_value / (1.0f + sycl::native::exp(-gate_value)) * up_value);
+}
+
+template <typename T>
+static void swiglu_clamp_sycl(const T * gate,
+ const T * up,
+ T * dst,
+ const int64_t k,
+ const int64_t n,
+ const int64_t o0,
+ const int64_t o1,
+ float limit,
+ dpct::queue_ptr stream) {
+ const int64_t num_blocks = (k + SYCL_GLU_BLOCK_SIZE - 1) / SYCL_GLU_BLOCK_SIZE;
+ stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE),
+ sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE)),
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+ swiglu_clamp_kernel(gate, up, dst, k, n, o0, o1, limit, item_ct1);
+ });
+}
+
+static void ggml_sycl_op_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ void * src0_d = src0->data;
+ void * src1_d = src1 ? src1->data : src0->data;
+ const int64_t src0_o = src0->nb[1];
+ const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
+ void * dst_d = dst->data;
+ const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
+ dpct::queue_ptr stream = ctx.stream();
+
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+ GGML_ASSERT(src0->nb[0] == ggml_element_size(src0));
+ GGML_ASSERT(ggml_is_contiguous(dst));
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == dst->type);
+ GGML_ASSERT(dst->ne[0] == nc);
+ GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0));
+
+ if (src1) {
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));
+ GGML_ASSERT(src1->ne[0] == nc);
+ GGML_ASSERT(src0->type == src1->type);
+ }
+
+ const int32_t swapped = ggml_get_op_params_i32(dst, 1);
+ const float limit = ggml_get_op_params_f32(dst, 3);
+
+ if (src0->type == GGML_TYPE_F16) {
+ sycl::half * src0_p = (sycl::half *) src0_d;
+ sycl::half * src1_p = (sycl::half *) src1_d;
+
+ if (!src1) {
+ src0_p += swapped ? nc : 0;
+ src1_p += swapped ? 0 : nc;
+ }
+
+ swiglu_clamp_sycl(src0_p, src1_p, (sycl::half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(sycl::half),
+ src1_o / sizeof(sycl::half), limit, stream);
+ } else {
+ float * src0_p = (float *) src0_d;
+ float * src1_p = (float *) src1_d;
+
+ if (!src1) {
+ src0_p += swapped ? nc : 0;
+ src1_p += swapped ? 0 : nc;
+ }
+
+ swiglu_clamp_sycl(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float),
+ src1_o / sizeof(float), limit, stream);
+ }
+}
+
static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {
return op_gelu_erf(x);
@@ -1295,6 +1391,11 @@ void ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
ggml_sycl_op_swiglu_oai(ctx, dst);
}
+void ggml_sycl_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
+ ggml_sycl_op_swiglu_clamp(ctx, dst);
+}
+
void ggml_sycl_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
ggml_sycl_op_geglu_erf(ctx, dst);
diff --git a/ggml/src/ggml-sycl/element_wise.hpp b/ggml/src/ggml-sycl/element_wise.hpp
index 67bf422d2f..d280066efb 100644
--- a/ggml/src/ggml-sycl/element_wise.hpp
+++ b/ggml/src/ggml-sycl/element_wise.hpp
@@ -77,6 +77,7 @@ void ggml_sycl_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
void ggml_sycl_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
void ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
+void ggml_sycl_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
void ggml_sycl_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index d58ffd00da..2b2d26cf23 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -5373,6 +5373,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg
case GGML_GLU_OP_SWIGLU_OAI:
ggml_sycl_swiglu_oai(ctx, dst);
break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ ggml_sycl_swiglu_clamp(ctx, dst);
+ break;
case GGML_GLU_OP_GEGLU_ERF:
ggml_sycl_geglu_erf(ctx, dst);
break;
@@ -6133,6 +6136,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
return ggml_is_contiguous_1(op->src[0]);
default:
return false;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 8fbb1359f4..394c84bf25 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1035,6 +1035,7 @@ struct vk_device_struct {
vk_pipeline pipeline_reglu[2];
vk_pipeline pipeline_swiglu[2];
vk_pipeline pipeline_swiglu_oai[2];
+ vk_pipeline pipeline_swiglu_clamp[2];
vk_pipeline pipeline_geglu_erf[2];
vk_pipeline pipeline_geglu_quick[2];
@@ -5748,6 +5749,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
CREATE_GLU(reglu)
CREATE_GLU(swiglu)
CREATE_GLU(swiglu_oai)
+ CREATE_GLU(swiglu_clamp)
CREATE_GLU(geglu_erf)
CREATE_GLU(geglu_quick)
#undef CREATE_GLU
@@ -11578,6 +11580,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
return ctx->device->pipeline_swiglu[dst->type == GGML_TYPE_F16];
case GGML_GLU_OP_SWIGLU_OAI:
return ctx->device->pipeline_swiglu_oai[dst->type == GGML_TYPE_F16];
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ return ctx->device->pipeline_swiglu_clamp[dst->type == GGML_TYPE_F16];
case GGML_GLU_OP_GEGLU_ERF:
return ctx->device->pipeline_geglu_erf[dst->type == GGML_TYPE_F16];
case GGML_GLU_OP_GEGLU_QUICK:
@@ -15883,6 +15887,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
ggml_vk_glu(ctx, compute_ctx, src0, src1, node);
break;
default:
@@ -18400,6 +18405,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
(op->src[0]->type == op->type) &&
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp b/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp
new file mode 100644
index 0000000000..dfe329759c
--- /dev/null
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp
@@ -0,0 +1,12 @@
+#version 450
+
+#include "glu_head.glsl"
+
+float op(float a, float b) {
+ float gate = min(a, p.limit);
+ float up = clamp(b, -p.limit, p.limit);
+
+ return gate / (1.0f + exp(-gate)) * up;
+}
+
+#include "glu_main.glsl"
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
index d375c2d127..f0610f4cd8 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
@@ -986,6 +986,8 @@ void process_shaders() {
string_to_spv("swiglu_f32", "swiglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("swiglu_oai_f16", "swiglu_oai.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("swiglu_oai_f32", "swiglu_oai.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
+ string_to_spv("swiglu_clamp_f16", "swiglu_clamp.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
+ string_to_spv("swiglu_clamp_f32", "swiglu_clamp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("geglu_erf_f16", "geglu_erf.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("geglu_erf_f32", "geglu_erf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("geglu_quick_f16","geglu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
index 7a67ccf4fc..a7ff36030f 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
@@ -3101,6 +3101,10 @@ class ggml_webgpu_shader_lib {
defines.push_back("OP_GEGLU_QUICK");
variant += "_geglu_quick";
break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ defines.push_back("OP_SWIGLU_CLAMP");
+ variant += "_swiglu_clamp";
+ break;
default:
GGML_ABORT("Unsupported GLU op");
}
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
index 2434848a55..b953118a7a 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
@@ -2835,7 +2835,7 @@ static webgpu_encoded_op ggml_webgpu_glu(webgpu_context & ctx,
(uint32_t) dst->ne[2],
(uint32_t) ((int32_t *) dst->op_params)[1], // swapped
ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 2)), // alpha, for swiglu_oai
- ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 3)), // limit, for swiglu_oai
+ ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 3)), // limit
};
std::vector<wgpu::BindGroupEntry> entries;
@@ -4483,6 +4483,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
case GGML_GLU_OP_SWIGLU:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
+ case GGML_GLU_OP_SWIGLU_CLAMP:
supports_op = op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
break;
case GGML_GLU_OP_SWIGLU_OAI:
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl
index d03f1c207d..6bbed5d3bf 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl
@@ -37,6 +37,14 @@ fn op(a: f32, b: f32) -> f32 {
return out_glu;
}
#endif
+#ifdef OP_SWIGLU_CLAMP
+fn op(a: DataType, b: DataType) -> DataType {
+ let limit = DataType(params.limit);
+ let gate = min(a, limit);
+ let up = clamp(b, -limit, limit);
+ return gate / (1.0 + exp(-gate)) * up;
+}
+#endif
#ifdef OP_GEGLU_ERF
const p_erf: DataType = 0.3275911;
const a1_erf: DataType = 0.254829592;
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index e0b615c07e..3bd3e3fe5e 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -1253,10 +1253,10 @@ static const char * GGML_GLU_OP_NAME[GGML_GLU_OP_COUNT] = {
"SWIGLU_OAI",
"GEGLU_ERF",
"GEGLU_QUICK",
+ "SWIGLU_CLAMP",
};
-static_assert(GGML_GLU_OP_COUNT == 6, "GGML_GLU_OP_COUNT != 6");
-
+static_assert(GGML_GLU_OP_COUNT == 7, "GGML_GLU_OP_COUNT != 7");
static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");
static_assert(sizeof(struct ggml_tensor)%GGML_MEM_ALIGN == 0, "ggml_tensor size must be a multiple of GGML_MEM_ALIGN");
@@ -3119,6 +3119,17 @@ struct ggml_tensor * ggml_swiglu_oai(
return result;
}
+struct ggml_tensor * ggml_swiglu_clamp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float limit) {
+ struct ggml_tensor * result = ggml_glu_impl(ctx, a, b, GGML_GLU_OP_SWIGLU_CLAMP, false);
+ ggml_set_op_params_f32(result, 3, limit);
+
+ return result;
+}
+
// ggml_norm
static struct ggml_tensor * ggml_norm_impl(
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 8fca8e1bc0..72db486cac 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1776,14 +1776,11 @@ ggml_tensor * llm_graph_context::build_ffn(
const float limit = hparams.swiglu_clamp_shexp[il];
constexpr float eps = 1e-6f;
if (limit > eps) {
- tmp = ggml_clamp(ctx0, tmp, -limit, limit);
- cb(tmp, "ffn_up_clamped", il);
-
if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {
- cur = ggml_clamp(ctx0, cur, -INFINITY, limit);
- cb(cur, "ffn_gate_clamped", il);
- cur = ggml_swiglu_split(ctx0, cur, tmp);
+ cur = ggml_swiglu_clamp(ctx0, cur, tmp, limit);
} else {
+ tmp = ggml_clamp(ctx0, tmp, -limit, limit);
+ cb(tmp, "ffn_up_clamped", il);
ggml_tensor * gate_act = ggml_silu(ctx0, cur);
cb(gate_act, "ffn_silu", il);
gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);
@@ -2173,14 +2170,11 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
const float limit = hparams.swiglu_clamp_exp[il];
constexpr float eps = 1e-6f;
if (limit > eps) {
- up = ggml_clamp(ctx0, up, -limit, limit);
- cb(up, "ffn_moe_up_clamped", il);
-
if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {
- cur = ggml_clamp(ctx0, cur, -INFINITY, limit);
- cb(cur, "ffn_moe_gate_clamped", il);
- cur = ggml_swiglu_split(ctx0, cur, up);
+ cur = ggml_swiglu_clamp(ctx0, cur, up, limit);
} else {
+ up = ggml_clamp(ctx0, up, -limit, limit);
+ cb(up, "ffn_moe_up_clamped", il);
ggml_tensor * gate_act = ggml_silu(ctx0, cur);
cb(gate_act, "ffn_moe_silu", il);
gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 4a7a062317..996b88db29 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -2243,6 +2243,63 @@ struct test_swiglu_oai : public test_case {
}
};
+struct test_swiglu_clamp : public test_case {
+ const ggml_type type;
+ const std::array<int64_t, 4> ne_a;
+ int v; // view (1 : non-contiguous a)
+ float limit;
+
+ std::string vars() override {
+ return VARS_TO_STR4(type, ne_a, v, limit);
+ }
+
+ test_swiglu_clamp(ggml_type type = GGML_TYPE_F32,
+ std::array<int64_t, 4> ne_a = {128, 2, 2, 2},
+ int v = 0,
+ float limit = 7.0f)
+ : type(type), ne_a(ne_a), v(v), limit(limit) {}
+
+ ggml_tensor * build_graph(ggml_context * ctx) override {
+ ggml_tensor * a;
+ ggml_tensor * b;
+ if (v & 1) {
+ auto ne = ne_a; ne[0] *= 3;
+ a = ggml_new_tensor(ctx, type, 4, ne.data());
+ ggml_set_param(a);
+ ggml_set_name(a, "a");
+
+ a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0);
+ ggml_set_name(a, "view_of_a");
+
+ b = ggml_new_tensor(ctx, type, 4, ne.data());
+ ggml_set_param(b);
+ ggml_set_name(b, "b");
+
+ b = ggml_view_4d(ctx, b, ne_a[0], ne_a[1], ne_a[2], ne_a[3], b->nb[1], b->nb[2], b->nb[3], 0);
+ ggml_set_name(b, "view_of_b");
+ } else {
+ a = ggml_new_tensor(ctx, type, 4, ne_a.data());
+ ggml_set_param(a);
+ ggml_set_name(a, "a");
+
+ b = ggml_new_tensor(ctx, type, 4, ne_a.data());
+ ggml_set_param(b);
+ ggml_set_name(b, "b");
+ }
+
+ ggml_tensor * out = ggml_swiglu_clamp(ctx, a, b, limit);
+ ggml_set_name(out, "out");
+
+ return out;
+ }
+
+ void initialize_tensors(ggml_context * ctx) override {
+ for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
+ init_tensor_uniform(t, -150.f, 150.f);
+ }
+ }
+};
+
// GGML_OP_GET_ROWS
struct test_get_rows : public test_case {
const ggml_type type;
@@ -6380,6 +6437,9 @@ struct test_mul_mat_vec_fusion : public test_case {
constexpr float alpha = 1.702f;
constexpr float limit = 7.0f;
out = ggml_swiglu_oai(ctx, ffn_gate, ffn_up, alpha, limit);
+ } else if (glu_op == GGML_GLU_OP_SWIGLU_CLAMP) {
+ constexpr float limit = 10.0f;
+ out = ggml_swiglu_clamp(ctx, ffn_gate, ffn_up, limit);
} else {
out = ggml_glu_split(ctx, ffn_gate, ffn_up, glu_op);
}
@@ -8376,8 +8436,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
for (int v : {0, 1}) {
for (int op = 0; op < GGML_GLU_OP_COUNT; op++) {
- if (op == GGML_GLU_OP_SWIGLU_OAI) {
- // SWIGLU_OAI is handled separately
+ if (op == GGML_GLU_OP_SWIGLU_OAI || op == GGML_GLU_OP_SWIGLU_CLAMP) {
continue;
}
@@ -8400,6 +8459,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
+ for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
+ for (int v : {0, 1}) {
+ for (float limit : {2.0f, 10.0f}) {
+ test_cases.emplace_back(new test_swiglu_clamp(type, { 128, 2, 2, 2 }, v, limit));
+ }
+ }
+ }
+
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_Q4_0}) {
test_cases.emplace_back(new test_get_rows(type, 300*256, 5, 4, 1, 2, false));
test_cases.emplace_back(new test_get_rows(type, 256, 80000, 70000, 2, 1, false));
@@ -10026,7 +10093,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
if (!with_gate && !with_bias) {
continue;
}
- for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {
+ for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_CLAMP}) {
if (!with_bias && glu_op == GGML_GLU_OP_SWIGLU_OAI) {
continue;
}
@@ -10041,7 +10108,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
use_id, 16, 8, b, with_bias, with_gate, with_lane_scale));
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,
use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}));
- if (!use_id && with_gate && !with_bias) {
+ if (!use_id && with_gate && !with_bias && glu_op != GGML_GLU_OP_SWIGLU_CLAMP) {
// small multi-token batches (speculative decoding / MTP verify)
for (int64_t m_batch : { 2, 4, 8 }) {
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, m_batch, 32, 256,
@@ -10056,6 +10123,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
+ for (bool b : {false, true}) {
+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_IQ2_S, GGML_GLU_OP_SWIGLU_CLAMP, 1, 32, 256,
+ true, 16, 8, b, false, true, false));
+ }
+
for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) {
for (bool with_norm : {false, true}) {
for (bool bias_probs : {false, true}) {