Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
45.6 KB
Raw Blame
// src/kernels/cuda/elementwise.cu - P2.S5's glue kernels.  See the header for why each exists.
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include <dpct/dpct.hpp>
#include "strata/sycl_queue.hpp"
#include "strata/sycl_doorbell.hpp"
#include "strata/kernels/elementwise.hpp"
#include "strata/kernels/dp4a.hpp"

#include "strata/kernels/bf16_bits.hpp"
#include "strata/kernels/f16_bits.hpp"

#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <atomic>
#include <chrono>

namespace strata::kernels {
namespace {

constexpr int THREADS = 256;

__dpct_inline__ void embedding_gather_kernel(const uint8_t *__restrict__ codes,
                                             const float *__restrict__ scales,
                                             const float *__restrict__ offsets,
                                             int64_t n, int code_bits,
                                             int code_bias, int group_elems,
                                             float *__restrict__ out) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int64_t i =
        (int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
        item_ct1.get_local_id(2);
    if (i >= n) return;
    const int per_byte = 8 / code_bits;
    const unsigned mask = (1u << code_bits) - 1u;
    const int code = (codes[i / per_byte] >> ((i % per_byte) * code_bits)) & mask;
    const int64_t group = i / group_elems;
    /*
    DPCT1013: The rounding mode could not be specified and the generated
    code may have different accuracy than the original code. Verify the
    correctness. SYCL math built-in function rounding mode is aligned with
    OpenCL C 1.2 standard.
    */
    const float product = (float)(code + code_bias) * scales[group];
    /*
    DPCT1013: The rounding mode could not be specified and the generated
    code may have different accuracy than the original code. Verify the
    correctness. SYCL math built-in function rounding mode is aligned with
    OpenCL C 1.2 standard.
    */
    out[i] = product + (offsets ? offsets[group] : 0.0f);
}

/// `ggml_compute_softplus_f32`: `log1p(exp(x))`, with the large-x branch that avoids overflow.
///
/// The branch is not cosmetic. `exp(89)` overflows f32 and `exp(20)` is already 4.85e8 where `log1p` loses
/// relative precision; above 20 the function is `x` to within f32 anyway.
__dpct_inline__ float softplus_dev(float x) {
    return x > 20.0f ? x : sycl::log1p(sycl::native::exp(x));
}

__dpct_inline__ void gdn_gate_kernel(const float *__restrict__ alpha,
                                     const float *__restrict__ dt,
                                     const float *__restrict__ ssm_a,
                                     float *__restrict__ gate, int64_t h_v) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int64_t i =
        (int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
        item_ct1.get_local_id(2);
    if (i >= h_v) return;
    const int64_t t = i / h_v;      // `n_tokens` is the leading dim; the real call has one token
    const int64_t h = i % h_v;
    gate[i] = softplus_dev(alpha[i] + dt[h]) * ssm_a[h];
    (void) t;
}

__dpct_inline__ void scale_kernel(float *__restrict__ x, int64_t n, float s) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int64_t i =
        (int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
        item_ct1.get_local_id(2);
    if (i < n) x[i] *= s;
}

// `dst[i] += src[i]`.  See the header: this is what lets R4's GPU half and CPU half run at the same
// time and still add up to one `parts` buffer.
__dpct_inline__ void add_kernel(float *__restrict__ dst,
                                const float *__restrict__ src, long long n) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const long long i =
        (long long)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
        item_ct1.get_local_id(2);
    if (i < n) dst[i] += src[i];
}

__dpct_inline__ void to_f16_kernel(const float *__restrict__ x,
                                   uint16_t *__restrict__ y, int64_t n) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int64_t i =
        (int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
        item_ct1.get_local_id(2);
    if (i < n) y[i] = f16_from_f32(x[i]);
}

__dpct_inline__ void to_bf16_kernel(const float *__restrict__ x,
                                    uint16_t *__restrict__ y, int64_t n) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int64_t i =
        (int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
        item_ct1.get_local_id(2);
    if (i < n) y[i] = bf16_from_f32(x[i]);
}

__dpct_inline__ void silu_kernel(float *__restrict__ x, int64_t n) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int64_t i =
        (int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
        item_ct1.get_local_id(2);
    if (i >= n) return;
    // DOUBLE then cast, matching `ref/gdn.py`'s numpy: its arrays are f32 but `np.exp` on an f32 array is
    // computed to f32 precision by a different algorithm than `expf`, and the reference is the oracle.  The
    // difference is in the last bits and this is one line.
    const double v = (double) x[i];
    x[i] = (float)(v / (1.0 + sycl::exp(-v)));
}

/// A bounded launch: a zero-length grid is illegal, and `n == 0` is a real call (an empty selection).
inline unsigned grid_for(int64_t n) { return (unsigned) ((n + THREADS - 1) / THREADS); }

/// One WARP per row, reduced through shuffles.  The rows here are short and few - (24, 256), (2, 256),
/// (4, 128) - so a block-per-row tree would spend its time in `__syncthreads` for 8 warps of 32, and the whole
/// call is 30 rows.  A warp reduction with NO shared memory and NO barrier is the shape that fits.
__dpct_inline__ void rms_norm_weighted_kernel(float *__restrict__ x,
                                              const float *__restrict__ w,
                                              int64_t rows, int64_t cols,
                                              float eps) {

    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int lane = item_ct1.get_local_id(2) & 31;
    const int64_t row =
        ((int64_t)item_ct1.get_group(2) * (item_ct1.get_local_range(2) >> 5)) +
        (item_ct1.get_local_id(2) >> 5);
    // **THE ROW GUARD IS NOT DECORATION, AND ITS ABSENCE WAS THE QSA BUG.**  The launcher rounds the grid up to
    // whole 4-warp blocks, so a call with `rows` = 2 - the QSA k-norm, the only non-multiple-of-4 row count in
    // the engine - runs EIGHT warps: rows 0 and 1 are the data, and rows 2 and 3 write 2*cols floats PAST the
    // end of `b.kcur`, which in the arena is exactly where `b.vcur` begins.  `b.vcur` was therefore silently
    // replaced by two rows of `rms_norm(...) * attn_k_norm`: a normalized vector with l2 ~20.6 against V's
    // ~5.7, which the attention then attended to.  `rope_neox_kernel` has had this guard all along.
    if (row >= rows) return;
    float* r = x + row * cols;
    float acc = 0.0f;
#pragma unroll
    for (int64_t c = lane; c < cols; c += 32) acc += r[c] * r[c];
    /*
DPCT1108: '__shfl_down_sync' was migrated with the experimental feature
masked sub_group function which may not be supported by all compilers or
runtimes. You may need to adjust the code.
*/
#pragma unroll
    for (int off = 16; off > 0; off >>= 1) acc +=
        dpct::experimental::shift_sub_group_left(
            0xFFFFFFFFu, sycl::ext::oneapi::this_work_item::get_sub_group(),
            acc, off);
    // The MEAN, not the sum: `ref/qsa.py::rms_norm` divides by `np.mean(np.square(x))`.  Broadcasting the
    // reciprocal from lane 0 keeps all 32 lanes on the same value - computing `rsqrt` per lane would be the
    // same number but a needless 32-way divergence in the last bit.
    float inv = 0.0f;
    if (lane == 0) inv = sycl::rsqrt(acc / (float)cols + eps);
    /*
    DPCT1108: '__shfl_sync' was migrated with the experimental feature masked
    sub_group function which may not be supported by all compilers or runtimes.
    You may need to adjust the code.
    */
    inv = dpct::experimental::select_from_sub_group(
        0xFFFFFFFFu, sycl::ext::oneapi::this_work_item::get_sub_group(), inv,
        0);
#pragma unroll
    for (int64_t c = lane; c < cols; c += 32)
        r[c] = (w ? r[c] * w[c] : r[c]) * inv;
}

void sync_if_needed(void *stream, const char *what) try {
    if (stream != nullptr) return;
    const dpct::err0 e =
        DPCT_CHECK_ERROR(dpct::get_current_device().queues_wait_and_throw());
}
catch (sycl::exception const &exc) {
  std::cerr << exc.what() << "Exception caught at file:" << __FILE__
            << ", line:" << __LINE__ << std::endl;
  std::exit(1);
}

bool check_launch(const char* what) {
    /*
    DPCT1010: SYCL uses exceptions to report errors and does not use the
    error codes. The cudaGetLastError function call was replaced with 0. You
    need to rewrite this code.
    */
    const dpct::err0 e = 0;

    return true;
}

}  // namespace

void embedding_gather(const uint8_t* codes, const float* scales, const float* offsets,
                      int64_t n, int code_bits, int code_bias, int group_elems,
                      float* out, void* stream) {
    if (n <= 0) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class embedding_gather_kernel_40b387>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid_for(n)) *
                                      sycl::range(1, 1, THREADS),
                                  sycl::range(1, 1, THREADS)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    embedding_gather_kernel(codes, scales, offsets, n,
                                            code_bits, code_bias, group_elems,
                                            out);
                });
    }
    check_launch("embedding_gather");
}

void gdn_gate(const float* alpha, const float* dt, const float* ssm_a, float* gate, int64_t n_tokens,
              int64_t h_v, void* stream) {
    if (n_tokens <= 0 || h_v <= 0) return;
    const int64_t n = n_tokens * h_v;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class gdn_gate_kernel_de22c0>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid_for(n)) *
                                      sycl::range(1, 1, THREADS),
                                  sycl::range(1, 1, THREADS)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    gdn_gate_kernel(alpha, dt, ssm_a, gate, h_v);
                });
    }
    check_launch("gdn_gate");
    sync_if_needed(stream, "gdn_gate");
}

void scale_inplace(float* x, int64_t n, float s, void* stream) {
    if (n <= 0) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class scale_kernel_501571>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid_for(n)) *
                                      sycl::range(1, 1, THREADS),
                                  sycl::range(1, 1, THREADS)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    scale_kernel(x, n, s);
                });
    }
    check_launch("scale_inplace");
    sync_if_needed(stream, "scale_inplace");
}

void add_inplace(float* dst, const float* src, int64_t n, void* stream) {
    if (n <= 0) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class add_kernel_95886e>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid_for(n)) *
                                      sycl::range(1, 1, THREADS),
                                  sycl::range(1, 1, THREADS)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    add_kernel(dst, src, n);
                });
    }
    check_launch("add_inplace");
    sync_if_needed(stream, "add_inplace");
}

void f32_to_f16_bulk(const float* x, uint16_t* y, int64_t n, void* stream) {
    if (n <= 0) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class to_f16_kernel_ba0968>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid_for(n)) *
                                      sycl::range(1, 1, THREADS),
                                  sycl::range(1, 1, THREADS)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    to_f16_kernel(x, y, n);
                });
    }
    check_launch("f32_to_f16_bulk");
    sync_if_needed(stream, "f32_to_f16_bulk");
}

void f32_to_bf16_bulk(const float* x, uint16_t* y, int64_t n, void* stream) {
    if (n <= 0) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class to_bf16_kernel_7b22d1>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid_for(n)) *
                                      sycl::range(1, 1, THREADS),
                                  sycl::range(1, 1, THREADS)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    to_bf16_kernel(x, y, n);
                });
    }
    check_launch("f32_to_bf16_bulk");
    sync_if_needed(stream, "f32_to_bf16_bulk");
}

void silu_inplace(float* x, int64_t n, void* stream) {
    if (n <= 0) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(
            strata::q_of(stream)->get_device(),
            {sycl::aspect::fp64});

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class silu_kernel_4d1386>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid_for(n)) *
                                      sycl::range(1, 1, THREADS),
                                  sycl::range(1, 1, THREADS)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    silu_kernel(x, n);
                });
    }
    check_launch("silu_inplace");
    sync_if_needed(stream, "silu_inplace");
}

/// THE DOORBELL.  One thread, one INCREMENT - the cost is the launch, and inside a graph that is paid once.
///
/// **IT INCREMENTS THE MEMORY, AND IT DOES NOT TAKE THE VALUE AS AN ARGUMENT.**  The first version did -
/// doorbell_ring(d_seq, *(h_seq) + 1u) - and that is a CLONED LITERAL: the expression is evaluated on the HOST
/// at CAPTURE time, so the graph rings 1 on every replay and a host waiting for a change waits forever.  It
/// would have looked like a working doorbell on the first token, which is the worst way for it to be wrong.
/// **THE FENCE IS NOT DECORATION, AND ITS ABSENCE WAS COSTING ~10 ms PER TOKEN ON THE HOST SIDE.**
///
/// The ring PUBLISHES three buffers the host is about to read - `h_x_f`, `h_ids`, `h_weights` - so the increment
/// must be ordered after their writes, or the host can observe the ring and then read a payload that has not
/// landed.  `__threadfence_system()` is what orders them, and it covers the HOST as well as the device, which is
/// the whole point: the reader is the CPU.
///
/// Without it the host loop was compensating with a driver call.  `session_loop` polled with `cudaEventQuery`
/// on EVERY spin iteration, because round 195 had measured that a memory-only spin never saw the datum flip -
/// and that measurement was right about the symptom and wrong about the cause.  The cause is this missing fence:
/// the write was not ordered into host-visible memory, so no amount of reading it would show it, and the driver
/// call was flushing the whole pipeline enough to make it appear.  A 10-22 us driver call per iteration is a
/// very expensive substitute for one fence instruction.
///
/// **THE STORE IS VOLATILE**, like `doorbell_publish_kernel`'s.  On RDNA4 (gfx1201) a plain store to mapped pinned
/// memory stays in the GPU's L2 until the stream is synchronized - the host never saw the ring (tests/hip/handoff:
/// 0 of 100 rings seen without a sync; volatile, a system-scope atomic store or a fence after the store: 100 of 100).
__dpct_inline__ void doorbell_ring_kernel(uint32_t *seq) {
    /*
    DPCT1078: Consider replacing memory_order::acq_rel with
    memory_order::seq_cst for correctness if strong memory order restrictions
    are needed.
    */
    sycl::atomic_fence(sycl::memory_order::acq_rel, sycl::memory_scope::system);
    strata::sys_store(seq, strata::sys_load(seq) + 1u);
#if defined(__HIPCC__)  // #697: HIP only; on RDNA4 the volatile store alone can sit in L2 until the stream syncs
    __threadfence_system();
#endif
}

__dpct_inline__ void doorbell_wait_kernel(const volatile uint32_t *flag,
                                          const volatile uint32_t *seq, uint32_t spin_max) {
    const uint32_t want = strata::sys_load(seq);
    for (uint32_t spin = 0; spin < spin_max && strata::sys_load(flag) != want; ++spin) strata_spin_pause();
    /*
    DPCT1078: Consider replacing memory_order::acq_rel with
    memory_order::seq_cst for correctness if strong memory order restrictions
    are needed.
    */
    sycl::atomic_fence(sycl::memory_order::acq_rel, sycl::memory_scope::system);
}

void doorbell_wait(const uint32_t* d_flag, const uint32_t* d_seq, void* stream) {
    if (d_flag == nullptr || d_seq == nullptr) return;
    const uint32_t spin_max = strata::spin_max(*strata::q_of(stream));
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class doorbell_wait_kernel_47b360>>(
                sycl::nd_range<3>(sycl::range(1, 1, 1), sycl::range(1, 1, 1)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    doorbell_wait_kernel(d_flag, d_seq, spin_max);
                });
    }
    check_launch("doorbell_wait");
}

/*
DPCT1052: SYCL does not support the member access for a volatile qualified
vector type. The volatile qualifier was removed. You may need to rewrite the
code.
*/
__dpct_inline__ void copy_from_mapped_kernel(sycl::float4 *__restrict__ dst,
                                             const sycl::float4 *src,
                                             int64_t n4) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    for (int64_t i =
             (int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
             item_ct1.get_local_id(2);
         i < n4; i += (int64_t)item_ct1.get_group_range(2) *
                      item_ct1.get_local_range(2)) {
        const sycl::float4 v = strata::load_mapped_float4(src + i);
        dst[i] = v;
    }
}

// the CPU rows of a verify window, skipping the rows the GPU plan computes itself (the
// pool writes +0.0 into those, so this writes +0.0 too): block = row, the plan's hit rows `dst[0, *count)`.
/*
DPCT1052: SYCL does not support the member access for a volatile qualified
vector type. The volatile qualifier was removed. You may need to rewrite the
code.
*/
__dpct_inline__ void copy_rows_from_mapped_kernel(
    sycl::float4 *__restrict__ dst, const sycl::float4 *src, int64_t row4,
    const int32_t *__restrict__ hit_rows, const int32_t *__restrict__ count) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int row = item_ct1.get_group(2);
    auto &hit = *sycl::ext::oneapi::group_local_memory_for_overwrite<int>(
        sycl::ext::oneapi::this_work_item::get_work_group<3>());
    if (item_ct1.get_local_id(2) == 0) {
        int h = 0;
        const int c = *count;
#pragma unroll
        for (int i = 0; i < c; ++i) h |= hit_rows[i] == row;
        hit = h;
    }
    item_ct1.barrier(sycl::access::fence_space::local_space);
    sycl::float4 *d = dst + (int64_t)row * row4;
    if (hit) {
#pragma unroll
        for (int64_t i = item_ct1.get_local_id(2); i < row4;
             i += item_ct1.get_local_range(2))
            d[i] = sycl::float4(0.0f, 0.0f, 0.0f, 0.0f);
    } else {
        /*
        DPCT1052: SYCL does not support the member access for a volatile
        qualified vector type. The volatile qualifier was removed. You may need
        to rewrite the code.
        */
        const sycl::float4 *sr = src + (int64_t)row * row4;
#pragma unroll
        for (int64_t i = item_ct1.get_local_id(2); i < row4;
             i += item_ct1.get_local_range(2))
            d[i] = strata::load_mapped_float4(sr + i);
    }
}
namespace {
__dpct_inline__ void scatter_rows_kernel(const sycl::float4 *__restrict__ src,
                                         sycl::float4 *dst,
                                         const int32_t *__restrict__ rows,
                                         int64_t w4) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int64_t r = item_ct1.get_group(2);
    const sycl::float4 *s = src + r * w4;
    sycl::float4 *d = dst + (int64_t)rows[r] * w4;
#pragma unroll
    for (int64_t i = item_ct1.get_local_id(2); i < w4;
         i += item_ct1.get_local_range(2)) d[i] = s[i];
}
}  // namespace
void scatter_rows_f32(const float* src, float* dst, const int32_t* rows, int64_t n, int64_t width, void* stream) {
    if (n <= 0) return;
    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
        std::fprintf(stderr, "scatter_rows_f32: width must be a multiple of 4 and both pointers 16-byte aligned\n");
        std::exit(1);
    }
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->submit([&](sycl::handler &cgh) {
                auto width_ct3 = width / 4;

                cgh.parallel_for<
                    dpct_kernel_name<class scatter_rows_kernel_7d83a1>>(
                    sycl::nd_range<3>(sycl::range(1, 1, (unsigned)n) *
                                          sycl::range(1, 1, 128),
                                      sycl::range(1, 1, 128)),
                    exp_props, [=](sycl::nd_item<3> item_ct1) {
                        scatter_rows_kernel((const sycl::float4 *)src,
                                            (sycl::float4 *)dst, rows,
                                            width_ct3);
                    });
            });
    }
}
void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
                           const int32_t* count, void* stream) {
    if (rows <= 0) return;
    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
        std::fprintf(stderr, "copy_rows_from_mapped: width must be a multiple of 4 and both pointers 16-byte aligned\n");
        std::exit(1);
    }
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->submit([&](sycl::handler &cgh) {
                auto width_ct2 = width / 4;

                cgh.parallel_for<dpct_kernel_name<
                    class copy_rows_from_mapped_kernel_9530d4>>(
                    sycl::nd_range<3>(sycl::range(1, 1, (unsigned)rows) *
                                          sycl::range(1, 1, 128),
                                      sycl::range(1, 1, 128)),
                    exp_props, [=](sycl::nd_item<3> item_ct1) {
                        copy_rows_from_mapped_kernel(
                            (sycl::float4 *)dst,
                            (const sycl::float4 *)src, width_ct2,
                            hit_rows, count);
                    });
            });
    }
}
void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream) {
    if (n <= 0) return;
    if ((n & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
        std::fprintf(stderr, "copy_from_mapped: n must be a multiple of 4 and both pointers 16-byte aligned\n");
        std::exit(1);
    }
    const int64_t n4 = n / 4;
    const int blocks = (int) ((n4 + 255) / 256 < 64 ? (n4 + 255) / 256 : 64);
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class copy_from_mapped_kernel_82951a>>(
                sycl::nd_range<3>(sycl::range(1, 1, blocks) *
                                      sycl::range(1, 1, 256),
                                  sycl::range(1, 1, 256)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    copy_from_mapped_kernel((sycl::float4 *)dst,
                                            (const sycl::float4 *)src,
                                            n4);
                });
    }
    check_launch("copy_from_mapped");
}

namespace {
// The published payload's checksum: the wrapping sum of every 32-bit word mixed with its position (murmur3's
// finalizer).  A plain sum of the words missed a reordering ([3, 7] -> [7, 3] in the expert ids) and cancelling
// changes; mixed, a stale payload passes only with a chance near 2^-32 per read.  It detects; it does not order -
// the host keeps reading until the payload it sees matches (doorbell_payload_ready).  Positions: x 0..n-1,
// ids n..n+k-1, weights n+k..n+2k-1.
inline uint32_t dbx_payload_mix(uint32_t word, uint32_t pos) {
    uint32_t h = word ^ (pos * 0x9E3779B9u);
    h ^= h >> 16; h *= 0x85EBCA6Bu; h ^= h >> 13; h *= 0xC2B2AE35u; h ^= h >> 16;
    return h;
}
inline uint32_t dbx_payload_sum(const float* x, int64_t n, const int32_t* ids, const float* w, int64_t k,
                                bool has_weights) {
    uint32_t s = 0;
    for (int64_t j = 0; j < n; ++j) s += dbx_payload_mix(sycl::bit_cast<uint32_t>(x[j]), (uint32_t) j);
    for (int64_t j = 0; j < k; ++j) s += dbx_payload_mix((uint32_t) ids[j], (uint32_t) (n + j));
    if (has_weights)
        for (int64_t j = 0; j < k; ++j) s += dbx_payload_mix(sycl::bit_cast<uint32_t>(w[j]), (uint32_t) (n + k + j));
    return s;
}
// STRATA_SYCL_A770_FAST=1 (opt-in): pair the activation stores without changing the checksum's word/position
// order. Odd sizes or unaligned mapped pointers retain the scalar path.
__dpct_inline__ uint32_t dbx_publish_x(const float* x, float* out, int n, int lane, int threads, bool pair, bool plain) {
    uint32_t part = 0;
    if (pair && !plain && ((uintptr_t)out & 7u) == 0) {
        for (int j = 2 * lane; j + 1 < n; j += 2 * threads) {
            const uint32_t a = sycl::bit_cast<uint32_t>(x[j]);
            const uint32_t b = sycl::bit_cast<uint32_t>(x[j + 1]);
            strata::sys_store_mapped(reinterpret_cast<uint64_t*>(out + j),
                                     (uint64_t)a | ((uint64_t)b << 32));
            part += dbx_payload_mix(a, (uint32_t)j) + dbx_payload_mix(b, (uint32_t)(j + 1));
        }
        if ((n & 1) && lane == 0) {
            strata::sys_store_mapped(plain, out + n - 1, x[n - 1]);
            part += dbx_payload_mix(sycl::bit_cast<uint32_t>(x[n - 1]), (uint32_t)(n - 1));
        }
    } else {
        for (int j = lane; j < n; j += threads) {
            strata::sys_store_mapped(plain, out + j, x[j]);
            part += dbx_payload_mix(sycl::bit_cast<uint32_t>(x[j]), (uint32_t)j);
        }
    }
    return part;
}
// The ring, its tag and its checksum, in slot r % 4 of the sequence word (elementwise.hpp).  Written last, after a
// system fence over the uncached payload stores.
__dpct_inline__ void dbx_ring_commit(uint32_t* seq, uint32_t part, bool group_ok, uint32_t ring) {
#if defined(__HIPCC__)  // #697: HIP only; on RDNA4 the volatile store alone can sit in L2 until the stream syncs
    (void) group_ok;
#endif
    if (group_ok) {
        strata::sys_store(seq + 2 + 2 * (ring % 4), part);
        strata::sys_store(seq + 1 + 2 * (ring % 4), ring);
        sycl::atomic_fence(sycl::memory_order::seq_cst, sycl::memory_scope::system);
        strata::sys_store(seq, ring);
    }
}
// The payload store (strata::sys_store_mapped(plain, ...)): uncached (sycl_doorbell.hpp) so a host that polls the ring while the kernel still runs reads a
// whole payload, or a plain store when every reader waits for the kernel to end (doorbell_plain_payload: the stepped
// verify window reads a segment's payload only after the segment has drained, and the end of a kernel makes its
// stores visible). The host checksum (doorbell_payload_ready) still guards every read.
std::atomic<bool> g_plain_payload{false};
}  // namespace

void doorbell_plain_payload(bool on) { g_plain_payload.store(on, std::memory_order_relaxed); }

__dpct_inline__ void doorbell_publish_kernel(const float *__restrict__ x,
                                             const int32_t *__restrict__ ids,
                                             const float *__restrict__ w, int n,
                                             int k, float *x_out,
                                             int32_t *ids_out, float *w_out,
                                             uint32_t *seq, bool plain, bool pair) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int i = (int) item_ct1.get_local_id(2);
    const int nt = (int) item_ct1.get_local_range(2);
    uint32_t part = dbx_publish_x(x, x_out, n, i, nt, pair, plain);
    if (i < k) {
        const int32_t id = ids[i];
        strata::sys_store_mapped(plain, ids_out + i, id);
        part += dbx_payload_mix((uint32_t) id, (uint32_t) (n + i));
        if (w_out != nullptr) {
            const float wv = w[i];
            strata::sys_store_mapped(plain, w_out + i, wv);
            part += dbx_payload_mix(sycl::bit_cast<uint32_t>(wv), (uint32_t) (n + k + i));
        }
    }
    const uint32_t sum = sycl::reduce_over_group(item_ct1.get_group(), part, sycl::plus<uint32_t>());
    sycl::atomic_fence(sycl::memory_order::seq_cst, sycl::memory_scope::system);
    item_ct1.barrier();
    if (i == 0) {
        const uint32_t next = strata::sys_load(seq) + 1u;
        dbx_ring_commit(seq, sum, true, next);
    }
}

__dpct_inline__ void doorbell_publish_res_kernel(
    const float *__restrict__ x, const int32_t *__restrict__ ids,
    const int32_t *__restrict__ d_res, int n_expert, int n, int k, float *x_out,
    int32_t *ids_out, uint32_t *seq, bool full, bool plain, bool pair) {
    // full (STRATA_DOORBELL_CHECK=1): the payload is published whole and the checksum covers x and the ids (this
    // variant publishes no weights). Otherwise d_res/n_expert decide whether x is copied, as before the checksum:
    // when every routed expert is resident the host never reads x, and an uncached copy of it costs the A750 ~10%
    // of decode.
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    if (!full) {
        int any_miss = 0;
        if ((int) item_ct1.get_local_id(2) < k) {
            const int32_t id = ids[item_ct1.get_local_id(2)];
            strata::sys_store_mapped(plain, ids_out + item_ct1.get_local_id(2), id);
            if (d_res == nullptr || id < 0 || id >= n_expert || d_res[id] < 0) any_miss = 1;
        }
        if ((item_ct1.barrier(), sycl::any_of_group(sycl::ext::oneapi::this_work_item::get_work_group<3>(), any_miss))) {
            for (int j = (int) item_ct1.get_local_id(2); j < n; j += (int) item_ct1.get_local_range(2))
                strata::sys_store_mapped(plain, x_out + j, x[j]);
        }
        sycl::atomic_fence(sycl::memory_order::seq_cst, sycl::memory_scope::system);
        item_ct1.barrier();
        if (item_ct1.get_local_id(2) == 0) {
            const uint32_t next = strata::sys_load(seq) + 1u;
            dbx_ring_commit(seq, 0u, true, next);
        }
        return;
    }
    const int i = (int) item_ct1.get_local_id(2);
    const int nt = (int) item_ct1.get_local_range(2);
    uint32_t part = dbx_publish_x(x, x_out, n, i, nt, pair, plain);
    if (i < k) {
        const int32_t id = ids[i];
        strata::sys_store_mapped(plain, ids_out + i, id);
        part += dbx_payload_mix((uint32_t) id, (uint32_t) (n + i));
    }
    const uint32_t sum = sycl::reduce_over_group(item_ct1.get_group(), part, sycl::plus<uint32_t>());
    sycl::atomic_fence(sycl::memory_order::seq_cst, sycl::memory_scope::system);
    item_ct1.barrier();
    if (i == 0) {
        const uint32_t next = strata::sys_load(seq) + 1u;
        dbx_ring_commit(seq, sum, true, next);
    }
}

// #649 (HIP, STRATA_DOORBELL_STORE=1): the same publish, but the ring is STORED (the step's own number, known at
// capture) instead of read-modify-written over PCIe - one store, no read of host memory from the GPU.
__dpct_inline__ void doorbell_publish_value_kernel(
    const float *__restrict__ x, const int32_t *__restrict__ ids,
    const float *__restrict__ w, int n, int k, float *x_out, int32_t *ids_out,
    float *w_out, uint32_t *seq, uint32_t value, bool plain, bool pair) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const int i = (int) item_ct1.get_local_id(2);
    const int nt = (int) item_ct1.get_local_range(2);
    uint32_t part = dbx_publish_x(x, x_out, n, i, nt, pair, plain);
    if (i < k) {
        const int32_t id = ids[i];
        strata::sys_store_mapped(plain, ids_out + i, id);
        part += dbx_payload_mix((uint32_t) id, (uint32_t) (n + i));
        const float wv = w[i];
        strata::sys_store_mapped(plain, w_out + i, wv);
        part += dbx_payload_mix(sycl::bit_cast<uint32_t>(wv), (uint32_t) (n + k + i));
    }
    const uint32_t sum = sycl::reduce_over_group(item_ct1.get_group(), part, sycl::plus<uint32_t>());
    sycl::atomic_fence(sycl::memory_order::seq_cst, sycl::memory_scope::system);
    item_ct1.barrier();
    if (i == 0) dbx_ring_commit(seq, sum, true, value);
}

void doorbell_publish_value(const float* x, const int32_t* ids, const float* weights, int64_t n, int64_t k,
                            float* x_out, int32_t* ids_out, float* weights_out, uint32_t* d_seq, uint32_t value,
                            void* stream) {
    if (k > 1024) { std::fprintf(stderr, "doorbell_publish: k too large\n"); std::exit(1); }
    const bool plain = strata::doorbell_plain_payload() || g_plain_payload.load(std::memory_order_relaxed);
    const bool pair = strata::a770_fast();
    /*
    DPCT1049: The work-group size passed to the SYCL kernel may exceed the
    limit. To get the device limit, query info::device::max_work_group_size.
    Adjust the work-group size if needed.
    */
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class doorbell_publish_value_kernel_9bdb22>>(
                sycl::nd_range<3>(sycl::range(1, 1, 1024),
                                  sycl::range(1, 1, 1024)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    doorbell_publish_value_kernel(x, ids, weights, (int)n,
                                                  (int)k, x_out, ids_out,
                                                  weights_out, d_seq, value, plain, pair);
                });
    }
}

void doorbell_publish(const float* x, const int32_t* ids, const float* weights, int64_t n, int64_t k, float* x_out,
                      int32_t* ids_out, float* weights_out, uint32_t* d_seq, void* stream) {
    if (k > 1024) { std::fprintf(stderr, "doorbell_publish: k too large\n"); std::exit(1); }
    const bool plain = strata::doorbell_plain_payload() || g_plain_payload.load(std::memory_order_relaxed);
    const bool pair = strata::a770_fast();
    /*
    DPCT1049: The work-group size passed to the SYCL kernel may exceed the
    limit. To get the device limit, query info::device::max_work_group_size.
    Adjust the work-group size if needed.
    */
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class doorbell_publish_kernel_8b5bad>>(
                sycl::nd_range<3>(sycl::range(1, 1, 1024),
                                  sycl::range(1, 1, 1024)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    doorbell_publish_kernel(x, ids, weights, (int)n, (int)k,
                                            x_out, ids_out, weights_out, d_seq, plain, pair);
                });
    }
    check_launch("doorbell_publish");
}

void doorbell_publish_res(const float* x, const int32_t* ids, const int32_t* d_res, int n_expert, int64_t n, int64_t k,
                          float* x_out, int32_t* ids_out, uint32_t* d_seq, void* stream) {
    if (k > 1024) { std::fprintf(stderr, "doorbell_publish_res: k too large\n"); std::exit(1); }
    static const bool full = [] { const char* v = std::getenv("STRATA_DOORBELL_CHECK"); return v != nullptr && v[0] == '1'; }();
    const bool fl = full;
    const bool plain = strata::doorbell_plain_payload() || g_plain_payload.load(std::memory_order_relaxed);
    const bool pair = strata::a770_fast();
    /*
    DPCT1049: The work-group size passed to the SYCL kernel may exceed the
    limit. To get the device limit, query info::device::max_work_group_size.
    Adjust the work-group size if needed.
    */
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class doorbell_publish_res_kernel_85b9b3>>(
                sycl::nd_range<3>(sycl::range(1, 1, 1024),
                                  sycl::range(1, 1, 1024)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    doorbell_publish_res_kernel(x, ids, d_res, n_expert, (int)n,
                                                (int)k, x_out, ids_out, d_seq, fl, plain, pair);
                });
    }
    check_launch("doorbell_publish_res");
}

bool doorbell_payload_ready(const uint32_t* h_seq, const float* h_x, int64_t n, const int32_t* h_ids,
                            const float* h_weights, int64_t k, uint32_t want) {
    const volatile uint32_t* w = h_seq + 1 + 2 * (want % 4);   // this ring's tag and checksum (elementwise.hpp)
    if (w[0] != want) return false;
    std::atomic_thread_fence(std::memory_order_acquire);
    // the two publish shapes: with the routing weights (doorbell_publish / _value) or without (doorbell_publish_res)
    return dbx_payload_sum(h_x, n, h_ids, h_weights, k, true) == w[1] ||
           dbx_payload_sum(h_x, n, h_ids, h_weights, k, false) == w[1];
}

bool doorbell_wait_payload(const uint32_t* h_seq, const float* h_x, int64_t n, const int32_t* h_ids,
                           const float* h_weights, int64_t k, uint32_t want, int timeout_ms) {
    // Opt-in (STRATA_DOORBELL_CHECK=1): on an Arc A750 (i915) the host-side checksum of the whole payload, read from
    // uncached host memory twice per CPU-expert layer, cost 15-20% of decode (5 interleaved rounds), and the stale
    // payload it guards against was not seen there. The ring tag is still published by the GPU.
    static const bool check = [] { const char* v = std::getenv("STRATA_DOORBELL_CHECK"); return v != nullptr && v[0] == '1'; }();
    if (!check) return true;
    if (doorbell_payload_ready(h_seq, h_x, n, h_ids, h_weights, k, want)) return true;
    const auto t0 = std::chrono::steady_clock::now();
    while (!doorbell_payload_ready(h_seq, h_x, n, h_ids, h_weights, k, want)) {
        for (int i = 0; i < 64; ++i) __builtin_ia32_pause();
        if (std::chrono::steady_clock::now() - t0 > std::chrono::milliseconds(timeout_ms)) return false;
    }
    return true;
}

__dpct_inline__ void copy_i32_from_mapped_kernel(int32_t *__restrict__ dst,
                                                 const volatile int32_t *src,
                                                 int n) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
#pragma unroll
    for (int i = item_ct1.get_local_id(2); i < n;
         i += item_ct1.get_local_range(2)) dst[i] = src[i];
}

#if !defined(__HIPCC__)
struct MappedCopies {
    MappedCopy c[8];
};

__dpct_inline__ void copy_from_mapped_multi_kernel(MappedCopies a) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    const MappedCopy c = a.c[item_ct1.get_group(1)];
    const int64_t i0 = (int64_t)item_ct1.get_group(2) *
                           item_ct1.get_local_range(2) +
                       item_ct1.get_local_id(2),
                  st = (int64_t)item_ct1.get_group_range(2) *
                       item_ct1.get_local_range(2);
    if ((c.words & 3) == 0 && ((uintptr_t) c.dst & 15) == 0 && ((uintptr_t) c.src & 15) == 0) {
        /*
        DPCT1052: SYCL does not support the member access for a volatile
        qualified vector type. The volatile qualifier was removed. You may need
        to rewrite the code.
        */
        const sycl::float4 *s = (const sycl::float4 *)c.src;
#pragma unroll
        for (int64_t i = i0; i < c.words / 4; i += st)(
            (sycl::float4 *)c.dst)[i] = const_cast<const sycl::float4 *>(s)[i];
    } else {
        const volatile int32_t* s = (const volatile int32_t*) c.src;
#pragma unroll
        for (int64_t i = i0; i < c.words; i += st)((int32_t *)c.dst)[i] = s[i];
    }
}

void copy_from_mapped_multi(const MappedCopy* copies, int n, void* stream) {
    if (n <= 0) return;
    if (n > 8) { std::fprintf(stderr, "copy_from_mapped_multi: at most 8 copies\n"); std::exit(1); }
    MappedCopies a;
    int64_t most = 0;
    for (int i = 0; i < n; ++i) {
        a.c[i] = copies[i];
        most = copies[i].words > most ? copies[i].words : most;
    }
    const int64_t units = (most + 3) / 4;
    const unsigned bx = (unsigned) ((units + 255) / 256 < 32 ? (units + 255) / 256 : 32);
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class copy_from_mapped_multi_kernel_66c71e>>(
                sycl::nd_range<3>(sycl::range(1, (unsigned)n, bx < 1 ? 1 : bx) *
                                      sycl::range(1, 1, 256),
                                  sycl::range(1, 1, 256)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    copy_from_mapped_multi_kernel(a);
                });
    }
    check_launch("copy_from_mapped_multi");
}
#endif

void copy_i32_from_mapped(int32_t* dst, const int32_t* src, int64_t n, void* stream) {
    if (n <= 0) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class copy_i32_from_mapped_kernel_a44acc>>(
                sycl::nd_range<3>(sycl::range(1, 1, 128),
                                  sycl::range(1, 1, 128)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    copy_i32_from_mapped_kernel(
                        dst, (const volatile int32_t *)src, (int)n);
                });
    }
    check_launch("copy_i32_from_mapped");
}

#if defined(__HIPCC__)   // AMD keeps one copy a launch
void copy_from_mapped_multi(const MappedCopy* copies, int n, void* stream) {
    for (int i = 0; i < n; ++i)
        copy_i32_from_mapped((int32_t*) copies[i].dst, (const int32_t*) copies[i].src, copies[i].words, stream);
}
#endif

void doorbell_ring(uint32_t* d_seq, void* stream) {
    if (d_seq == nullptr) return;
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<dpct_kernel_name<class doorbell_ring_kernel_b862a4>>(
                sycl::nd_range<3>(sycl::range(1, 1, 1), sycl::range(1, 1, 1)),
                exp_props, [=](sycl::nd_item<3> item_ct1) {
                    doorbell_ring_kernel(d_seq);
                });
    }
    check_launch("doorbell_ring");
    sync_if_needed(stream, "doorbell_ring");
}

void rms_norm_weighted(float* x, const float* w, int64_t rows, int64_t cols, float eps, void* stream) {
    if (rows <= 0 || cols <= 0) return;
    // 4 warps per block, so a row count that is not a multiple of 4 wastes at most 3 warps rather than
    // launching a block per row for a 2-row call.
    const unsigned warps_per_block = 4;
    const unsigned grid = (unsigned) ((rows + warps_per_block - 1) / warps_per_block);
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        strata::q_of(stream)
            ->parallel_for<
                dpct_kernel_name<class rms_norm_weighted_kernel_b603f5>>(
                sycl::nd_range<3>(sycl::range(1, 1, grid) *
                                      sycl::range(1, 1, warps_per_block * 32),
                                  sycl::range(1, 1, warps_per_block * 32)),
                exp_props,
                [=](sycl::nd_item<3> item_ct1)
                    [[sycl::reqd_sub_group_size(32)]] {
                        rms_norm_weighted_kernel(x, w, rows, cols, eps);
                    });
    }
    check_launch("rms_norm_weighted");
    sync_if_needed(stream, "rms_norm_weighted");
}

}  // namespace strata::kernels