LLM inference in C/C++
C++ ggml
sycl: *glu flat path (#26354)
Titaniumtown6b5c2efb4e2f
diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cppindex 3cd055494e..0e707d531b 100644--- a/ggml/src/ggml-sycl/element_wise.cpp+++ b/ggml/src/ggml-sycl/element_wise.cpp@@ -420,53 +420,31 @@ static void clamp(const T * x, T * dst, const float min, const float max, const}}-template<typename T>-static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {- SYCL_GLOBAL_ID_LOOP(k, item_ct1) {- const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);- const int64_t j0 = rc.x() * o0 + rc.y();- const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();- dst[i] = op_gelu(x[j0]) * g[j1];- }-}--template<typename T>-static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {+template<typename T, typename F>+static void unary_gated_op_flat_kernel(const T * x, const T * g, T * dst, const uint64_t k, const sycl::nd_item<1> & item_ct1, F func) {SYCL_GLOBAL_ID_LOOP(k, item_ct1) {- const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);- const int64_t j0 = rc.x() * o0 + rc.y();- const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();- dst[i] = op_relu(x[j0]) * g[j1];+ dst[i] = func(x[i]) * g[i];}}-template<typename T>-static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {- SYCL_GLOBAL_ID_LOOP(k, item_ct1) {- const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);- const int64_t j0 = rc.x() * o0 + rc.y();- const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();- dst[i] = op_silu(x[j0]) * g[j1];- }-}--template<typename T>-static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {- SYCL_GLOBAL_ID_LOOP(k, item_ct1) {- const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);- const int64_t j0 = rc.x() * o0 + rc.y();- const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();- dst[i] = op_gelu_erf(x[j0]) * g[j1];- }-}+template<typename T, typename F>+static void unary_gated_op_generic_kernel(+ const T * x,+ const T * g,+ T * dst,+ const uint64_t k,+ const sycl::uint3 n_fd,+ const uint64_t o0,+ const uint64_t o1,+ const sycl::nd_item<1> & item_ct1,+ F func) {-template<typename T>-static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {+ // rows of n columns at strides o0 and o1: two halves of one fused tensor, or two tensorsSYCL_GLOBAL_ID_LOOP(k, item_ct1) {const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);const int64_t j0 = rc.x() * o0 + rc.y();const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();- dst[i] = op_gelu_quick(x[j0]) * g[j1];+ dst[i] = func(x[j0]) * g[j1];}}@@ -670,6 +648,35 @@ static inline void ggml_sycl_op_unary(});}+template<typename F>+static inline void ggml_sycl_op_unary_gated(+ ggml_backend_sycl_context & ctx, ggml_tensor * dst, F func) {++ dispatch_ggml_sycl_op_fused_glu(ctx, dst,+ [func](const auto * x_ptr, const auto * g_ptr, auto * dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {++ const uint32_t num_blocks = (uint32_t) ceil_div(k, SYCL_GLU_BLOCK_SIZE);+ const sycl::nd_range<1> launch_range(num_blocks * sycl::range<1>(SYCL_GLU_BLOCK_SIZE),+ sycl::range<1>(SYCL_GLU_BLOCK_SIZE));++ // o0 == n and o1 == n make the index math the identity, so index flat+ // note: not ggml_is_contiguous - a fused [gate|up] src0 is contiguous with o0 == 2n+ if (o0 == n && o1 == n) {+ main_stream->parallel_for(launch_range,+ [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ unary_gated_op_flat_kernel(x_ptr, g_ptr, dst_ptr, k, item_ct1, func);+ });+ } else {+ // launch-invariant divisor, and only this path needs it+ const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);+ main_stream->parallel_for(launch_range,+ [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ unary_gated_op_generic_kernel(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1, func);+ });+ }+ });+}+static inline void ggml_sycl_op_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {GGML_ASSERT(dst->type == GGML_TYPE_F32);@@ -967,42 +974,21 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor}static inline void ggml_sycl_op_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {- ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,- [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {- const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);- const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);- main_stream->parallel_for(- sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),- sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {- gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);- });- });+ ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {+ return op_gelu(x);+ });}static inline void ggml_sycl_op_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {- ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,- [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {- const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_RELU_BLOCK_SIZE); // Using RELU block size for reglu- const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);- main_stream->parallel_for(- sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_RELU_BLOCK_SIZE)),- sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {- gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);- });- });+ ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {+ return op_relu(x);+ });}static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {- ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,- [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {- const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_SILU_BLOCK_SIZE); // Using SILU block size for swiglu- const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);- main_stream->parallel_for(- sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_SILU_BLOCK_SIZE)),- sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {- gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);- });- });+ ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {+ return op_silu(x);+ });}__dpct_inline__ float ggml_sycl_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) {@@ -1097,29 +1083,15 @@ void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst)}static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {- ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,- [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {- const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);- const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);- main_stream->parallel_for(- sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),- sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {- gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);- });- });+ ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {+ return op_gelu_erf(x);+ });}static inline void ggml_sycl_op_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {- ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,- [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {- const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);- const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);- main_stream->parallel_for(- sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),- sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {- gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);- });- });+ ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {+ return op_gelu_quick(x);+ });}diff --git a/ggml/src/ggml-sycl/presets.hpp b/ggml/src/ggml-sycl/presets.hppindex 502e3b6105..789f3ef0f2 100644--- a/ggml/src/ggml-sycl/presets.hpp+++ b/ggml/src/ggml-sycl/presets.hpp@@ -20,8 +20,6 @@#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses#define SYCL_COL2IM_1D_BLOCK_SIZE 256-#define SYCL_GELU_BLOCK_SIZE 256-#define SYCL_SILU_BLOCK_SIZE 256#define SYCL_TANH_BLOCK_SIZE 256#define SYCL_RELU_BLOCK_SIZE 256#define SYCL_HARDSIGMOID_BLOCK_SIZE 256diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 8cb5989358..fbbfee6302 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9747,6 +9747,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {std::vector<std::unique_ptr<test_case>> test_cases;+ // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands+ // note: same bytes either way, so a backend that indexes them differently shows it here+ for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {+ for (int64_t n_tokens : {512, 2048}) {+ test_cases.emplace_back(new test_glu(GGML_GLU_OP_SWIGLU, type, { 2*17408, n_tokens, 1, 1 }, 0, false));+ test_cases.emplace_back(new test_glu_split(GGML_GLU_OP_SWIGLU, type, { 17408, n_tokens, 1, 1 }, 0));+ }+ }+// Conv2d: K=CRS=NPQ=4096 matmul performanceuint32_t iwh_idx = 0;uint32_t kwh_idx = 1;