LLM inference in C/C++
C++ ggml
cpu: add F16 SWIGLU_OAI reference implementation and validation (#29200)
cqderek5a638aedb7c4
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex c7e7c47a79..3b39683d5c 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -3548,6 +3548,73 @@ static void ggml_compute_forward_swiglu_oai_f32(}}+static void ggml_compute_forward_swiglu_oai_f16(+ const ggml_compute_params * params,+ ggml_tensor * dst) {++ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ char * src0_d = (char *) src0->data;+ char * src1_d = (char *) (src1 ? src1->data : src0->data);+ const size_t src0_o = src0->nb[1];+ const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];++ GGML_ASSERT(ggml_is_contiguous_1(src0));+ GGML_ASSERT(ggml_is_contiguous_1(dst));++ if (src1) {+ GGML_ASSERT(ggml_is_contiguous_1(src1));+ GGML_ASSERT(src0->type == src1->type);+ }++ const int ith = params->ith;+ const int nth = params->nth;++ const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;+ const int nr = ggml_nrows(src0);++ GGML_ASSERT(dst->ne[0] == nc);+ GGML_ASSERT(ggml_nrows(dst) == nr);++ const int32_t swapped = ggml_get_op_params_i32(dst, 1);+ const float alpha = ggml_get_op_params_f32(dst, 2);+ const float limit = ggml_get_op_params_f32(dst, 3);++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int i1 = ir0; i1 < ir1; i1++) {+ ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);+ ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);+ ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));++ if (!src1) {+ src0_p += swapped ? nc : 0;+ src1_p += swapped ? 0 : nc;+ }++ for (int k = 0; k < nc; k++) {+ const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);+ const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);+ const float out_glu = x / (1.f + expf(alpha * (-x)));+ dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));+ }++#ifndef NDEBUG+ for (int k = 0; k < nc; k++) {+ const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);+ GGML_UNUSED(x);+ assert(!isnan(x));+ assert(!isinf(x));+ }+#endif // NDEBUG+ }+}+static void ggml_compute_forward_swiglu_oai(const ggml_compute_params * params,ggml_tensor * dst) {@@ -3559,6 +3626,10 @@ static void ggml_compute_forward_swiglu_oai({ggml_compute_forward_swiglu_oai_f32(params, dst);} break;+ case GGML_TYPE_F16:+ {+ ggml_compute_forward_swiglu_oai_f16(params, dst);+ } break;default:{GGML_ABORT("fatal error");diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 9fe618f855..7c45d0d1fb 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5474,6 +5474,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g(ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_CLAMP)) {return false;}+ if (op->src[0]->type == GGML_TYPE_F16 && ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI) {+ return false;+ }return ggml_is_contiguous_1(op->src[0]);default:return false;diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cppindex 7eea81d96e..2c206a597d 100644--- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp+++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp@@ -75,6 +75,15 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {const float alpha = reinterpret_cast<const float *>(params)[2];const float limit = reinterpret_cast<const float *>(params)[3];+ // Compute in f32: alpha_const/one are f32, and Swish/Add need matching input types.+ auto output_type = context.get_output_type();+ if (src0.get_element_type() != ov::element::f32) {+ src0 = std::make_shared<ov::op::v0::Convert>(src0, ov::element::f32);+ }+ if (src1.get_element_type() != ov::element::f32) {+ src1 = std::make_shared<ov::op::v0::Convert>(src1, ov::element::f32);+ }+auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha});auto out_glu = std::make_shared<ov::op::v4::Swish>(gate, alpha_const);@@ -82,7 +91,10 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f});auto up_plus_one = std::make_shared<ov::op::v1::Add>(up, one);- auto res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);+ ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);+ if (res.get_element_type() != output_type) {+ res = std::make_shared<ov::op::v0::Convert>(res, output_type);+ }return rename_outputs_with_suffix({res}, context.get_name());}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex e61823084a..93b30d8d25 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -2308,6 +2308,12 @@ struct test_swiglu_oai : public test_case {init_tensor_uniform(t, -150.f, 150.f);}}++ double max_nmse_err() override {+ // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the+ // natural roundoff drift from the exp()/clamp() chain computed in F16.+ return type == GGML_TYPE_F16 ? 5e-6 : 1e-7;+ }};struct test_swiglu_clamp : public test_case {@@ -2365,6 +2371,12 @@ struct test_swiglu_clamp : public test_case {init_tensor_uniform(t, -150.f, 150.f);}}++ double max_nmse_err() override {+ // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the+ // natural roundoff drift from the sigmoid()/clamp() chain computed in F16.+ return type == GGML_TYPE_F16 ? 5e-6 : 1e-7;+ }};// GGML_OP_GET_ROWS@@ -9747,10 +9759,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}}- for (int v : {0, 1}) {- for (float alpha : {.5f, 1.702f}) {- for (float limit : {2.0f, 7.0f}) {- test_cases.emplace_back(new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit));+ for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {+ for (int v : {0, 1}) {+ for (float alpha : {.5f, 1.702f}) {+ for (float limit : {2.0f, 7.0f}) {+ test_cases.emplace_back(new test_swiglu_oai(type, { 128, 2, 2, 2 }, v, alpha, limit));+ }}}}