Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
50.6 KB
Raw Blame
// src/kernels/cuda/qsa_decode_attn.cu - see include/strata/kernels/qsa_decode_attn.hpp.
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include "strata/core/geometry.hpp"
#include <dpct/dpct.hpp>
#include "strata/sycl_queue.hpp"
#include "strata/kernels/qsa_decode_attn.hpp"
#include "strata/kernels/kv_q8.hpp"
#include "strata/kernels/kv_q4.hpp"

#include <cfloat>
#include <cstdio>
#include <cstdlib>
#include <cmath>

namespace strata::kernels {
namespace {

constexpr int HD = 256;          // head_dim
constexpr int G = 12;            // query heads per KV head (24 / 2)
static_assert(HD == strata::core::kCompiledGeometry.head_dim && G == strata::core::kCompiledGeometry.n_head / strata::core::kCompiledGeometry.n_head_kv, "qsa_decode_attn is compiled for core/geometry.hpp");
constexpr int CHUNK = 64;        // cells per block
constexpr int THREADS = 256;
constexpr int WARPS = THREADS / 32;

__dpct_inline__ float warp_sum(float v) {
#pragma unroll
    /*
    DPCT1108: '__shfl_xor_sync' was migrated with the experimental feature
    masked sub_group function which may not be supported by all compilers or
    runtimes. You may need to adjust the code.
    */
    for (int o = 16; o > 0; o >>= 1) v +=
        dpct::experimental::permute_sub_group_by_xor(
            0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(), v,
            o);
    return v;
}
__dpct_inline__ float warp_max(float v) {
#pragma unroll
    /*
    DPCT1108: '__shfl_xor_sync' was migrated with the experimental feature
    masked sub_group function which may not be supported by all compilers or
    runtimes. You may need to adjust the code.
    */
    for (int o = 16; o > 0; o >>= 1) v = sycl::fmax(
        v, dpct::experimental::permute_sub_group_by_xor(
               0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(),
               v, o));
    return v;
}

// 8 consecutive values of one cell's key or value row for KV head `kvh`, dimensions [d0, d0+8).
// Per-format bodies; `load8` below is the KV_MODE dispatcher. value=false is the K side, true the V side.
__dpct_inline__ void load8_f16(const QsaAttnPools &p, bool value, long long row,
                               int d0, float *out) {
    const uint16_t* base = (value ? p.v_pool : p.k_pool) + row * HD + d0;
    const sycl::uint4 raw = *reinterpret_cast<const sycl::uint4 *>(base);
    const sycl::half2 *h2 = reinterpret_cast<const sycl::half2 *>(&raw);
#pragma unroll
    for (int j = 0; j < 4; ++j) {
        const sycl::float2 f =
            h2[j].template convert<float, sycl::rounding_mode::automatic>();
        out[2 * j] = f.x();
        out[2 * j + 1] = f.y();
    }
}
__dpct_inline__ void load8_q8(const QsaAttnPools &p, bool value, long long row,
                              int d0, float *out) {
    const int8_t* codes = (value ? p.v_q : p.k_q) + row * HD + d0;
    const uint16_t sbits = (value ? p.v_scale : p.k_scale)[row * (HD / KV_Q8_GROUP) + d0 / KV_Q8_GROUP];
    const float sc = sycl::vec<sycl::half, 1>(
                         sycl::bit_cast<sycl::half, unsigned short>(sbits))
                         .convert<float, sycl::rounding_mode::automatic>()[0];
    const sycl::uint2 raw = *reinterpret_cast<const sycl::uint2 *>(codes);
    const int8_t* c = reinterpret_cast<const int8_t*>(&raw);
#pragma unroll
    for (int j = 0; j < 8; ++j) out[j] = (float) c[j] * sc;
}
__dpct_inline__ void load8_q4(const QsaAttnPools &p, bool value, long long row,
                              int d0, float *out) {
    constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
    const int b = d0 / QK4_0;
    const int rem = d0 % QK4_0;
    const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((value ? p.v_q4 : p.k_q4) + row * bytes_per_head) + b;
    const float d = sycl::vec<sycl::half, 1>(
                        sycl::bit_cast<sycl::half, unsigned short>(blk->d))
                        .convert<float, sycl::rounding_mode::automatic>()[0];
    const int j = (rem == 0 || rem == 16) ? 0 : 8;
    const uint8_t* bytes = blk->qs + j;
    if (rem < 16) {
#pragma unroll
        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] & 0x0F) - 8) * d;
    } else {
#pragma unroll
        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] >> 4) - 8) * d;
    }
}
template <int KV_MODE>
__dpct_inline__ void load8(const QsaAttnPools &p, bool value, long long row,
                           int d0, float *out) {
    if constexpr (KV_MODE == 0) load8_f16(p, value, row, d0, out);
    else if constexpr (KV_MODE == 1) load8_q8(p, value, row, d0, out);
    else if constexpr (KV_MODE == 3) {
        // hybrid K8V4: both sides are defined - K unrotated INT8, V rotated Q4_0 - so a value=true call
        // reads the Q4_0 pool instead of dereferencing the null v_q (no call site does today; PR review)
        if (value) load8_q4(p, value, row, d0, out);
        else load8_q8(p, value, row, d0, out);
    } else load8_q4(p, value, row, d0, out);
}

template <int KV_MODE, bool LANE_CELL = false, int RS = 0>
/*
DPCT1110: The total declared local variable size in device function
attn_chunk_kernel exceeds 128 bytes and may cause high register pressure.
Consult with your hardware vendor to find the total register size available and
adjust the code, or use smaller sub-group size to avoid high register pressure.
*/
__dpct_inline__ void
attn_chunk_kernel(const float *__restrict__ q, QsaAttnPools p,
                  const int32_t *__restrict__ ids,
                  const int32_t *__restrict__ step, int n_kv_heads,
                  int page_size, float scale, float *__restrict__ part_acc,
                  float *__restrict__ part_m, float *__restrict__ part_l,
                  int n_chunks, int cap = 0, long long scratch_stride = 0) {
    // batched form: query blockIdx.z, with its own q row, selection, step and scratch
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    q += (size_t)item_ct1.get_group(0) * (size_t)(n_kv_heads * G) * HD;
    ids += (size_t)item_ct1.get_group(0) * (size_t)cap;
    step += (size_t)item_ct1.get_group(0) * kStepCount;
    part_acc += (size_t)item_ct1.get_group(0) * (size_t)scratch_stride;
    part_m += (size_t)item_ct1.get_group(0) * (size_t)scratch_stride;
    part_l += (size_t)item_ct1.get_group(0) * (size_t)scratch_stride;
    auto &sq =
        *sycl::ext::oneapi::group_local_memory_for_overwrite<float[G][HD]>(
            sycl::ext::oneapi::this_work_item::get_work_group<
                3>()); // 12 KB: this KV head's query heads
    auto &sp =
        *sycl::ext::oneapi::group_local_memory_for_overwrite<float[G * CHUNK]>(
            sycl::ext::oneapi::this_work_item::get_work_group<
                3>()); // scores, then probabilities
    auto &srow =
        *sycl::ext::oneapi::group_local_memory_for_overwrite<long long[CHUNK]>(
            sycl::ext::oneapi::this_work_item::get_work_group<
                3>()); // pool row of each cell (page, kv head, slot)
    /*
    DPCT1098: The '*' expression is used instead of the __ldg call. These
    two expressions do not provide the exact same functionality. Check the
    generated code for potential precision and/or performance issues.
    */
    // the scores and probabilities: [head][cell], or - RS - [cell][head] so the value phase reads a cell's 12 probabilities as 3 vectors
    const auto spi = [](int h, int c) { return RS != 0 ? c * G + h : h * CHUNK + c; };
    const int n_ids = *(step + kStepWidth);
    const int chunk = item_ct1.get_group(2), kvh = item_ct1.get_group(1);
    const int t = item_ct1.get_local_id(2), lane = t & 31, warp = t >> 5;
    const int c0 = chunk * CHUNK;
    const int n_here = sycl::min(CHUNK, n_ids - c0);
    const int slot = kvh * n_chunks + chunk;
    if (n_here <= 0) {
        if (t < G) { part_m[slot * G + t] = -FLT_MAX; part_l[slot * G + t] = 0.0f; }
        return;
    }
#pragma unroll
    for (int i = t; i < G * HD; i += THREADS)
        sq[i / HD][i % HD] = q[(size_t)(kvh * G) * HD + i];
    if (t < CHUNK) {
        long long r = -1;
        if (t < n_here) {
            const int cell = ids[c0 + t];
            const long long page = (long long) p.page_table[cell / page_size];
            // a block the KV streaming could not make resident keeps page -1 (ctl[3]); its cells are masked
            // (score -FLT_MAX, weight 0) instead of being read from before the pool.
            if (page >= 0) r = (page * n_kv_heads + kvh) * page_size + (cell % page_size);
        }
        srow[t] = r;
    }
    /*
    DPCT1065: Consider replacing sycl::nd_item::barrier() with
    sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
    performance if there is no access to global memory.
    */
    item_ct1.barrier();
    if constexpr (LANE_CELL) {
        // S25 (STRATA_ATTN_LANECELL=1): thread t scores cell t % 64 for heads 3 (t / 64) .. +2 by itself - the same
        // 8-dimension partial per former lane (the identical expression), then the former warp_sum's butterfly as
        // lane 0 ran it (a[l] += a[l + o] for o = 16 .. 1), so every score is bit-identical without 12 shuffle
        // reductions per cell
        constexpr int HPT = G * CHUNK / THREADS;   // heads per thread (3)
        const int c = t % CHUNK, h0 = (t / CHUNK) * HPT;
        if (c >= n_here || srow[c] < 0) {
#pragma unroll
            for (int j = 0; j < HPT; ++j) sp[spi(h0 + j, c)] = -FLT_MAX;
        } else {
            float a[HPT][32];
#pragma unroll
            for (int l = 0; l < 32; ++l) {
                float k8[8];
                load8<KV_MODE>(p, false, srow[c], l * 8, k8);
#pragma unroll
                for (int j = 0; j < HPT; ++j) {
                    const sycl::float4 qa =
                        *reinterpret_cast<const sycl::float4 *>(
                            &sq[h0 + j][l * 8]);
                    const sycl::float4 qb =
                        *reinterpret_cast<const sycl::float4 *>(
                            &sq[h0 + j][l * 8 + 4]);
                    float s = k8[0] * qa.x() + k8[1] * qa.y() + k8[2] * qa.z() +
                              k8[3] * qa.w() + k8[4] * qb.x() + k8[5] * qb.y() +
                              k8[6] * qb.z() + k8[7] * qb.w();
                    a[j][l] = s;
                }
            }
#pragma unroll
            for (int j = 0; j < HPT; ++j) {
#pragma unroll
                for (int o = 16; o > 0; o >>= 1)
#pragma unroll
                    for (int l = 0; l < o; ++l) a[j][l] = a[j][l] + a[j][l + o];
                sp[spi(h0 + j, c)] = a[j][0] * scale;
            }
        }
    } else
    // scores: each warp takes cells warp, warp+8, ...; each lane holds 8 of the 256 dimensions.
    for (int c = warp; c < CHUNK; c += WARPS) {
        if (c >= n_here || srow[c] < 0) {
            if (lane < G) sp[spi(lane, c)] = -FLT_MAX;
            continue;
        }
        float k8[8];
        load8<KV_MODE>(p, false, srow[c], lane * 8, k8);
        if constexpr (RS == 1) {
            // 151-b70: the 12 heads' 32-lane butterfly sums as ONE reduce-scatter - 16 shuffles instead of 60.  Each stage pairs the
            // same lanes in the same order as warp_sum (xor 16, 8, 4, 2, 1) and adds own + partner, so every head's sum is the
            // same tree and the same bits; the heads are padded to 16, halved at each stage (the lane keeps the half its bit
            // selects, sends the other), and after the xor-2 stage lane 2h + {0, 1} holds head h's partial pair; xor 1 finishes it.
            float part[16];
#pragma unroll
            for (int h = 0; h < G; ++h) {
                const sycl::float4 qa =
                    *reinterpret_cast<const sycl::float4 *>(&sq[h][lane * 8]);
                const sycl::float4 qb =
                    *reinterpret_cast<const sycl::float4 *>(&sq[h][lane * 8 + 4]);
                part[h] = k8[0] * qa.x() + k8[1] * qa.y() + k8[2] * qa.z() +
                          k8[3] * qa.w() + k8[4] * qb.x() + k8[5] * qb.y() +
                          k8[6] * qb.z() + k8[7] * qb.w();
            }
#pragma unroll
            for (int h = G; h < 16; ++h) part[h] = 0.0f;
            const auto xr = [](float v, int o) {
                return dpct::experimental::permute_sub_group_by_xor(
                    0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(), v, o);
            };
            float c8[8];
            const bool b16 = (lane & 16) != 0, b8 = (lane & 8) != 0, b4 = (lane & 4) != 0, b2 = (lane & 2) != 0;
#pragma unroll
            for (int i = 0; i < 8; ++i) {
                const float send = b16 ? part[i] : part[8 + i];
                const float keep = b16 ? part[8 + i] : part[i];
                c8[i] = keep + xr(send, 16);
            }
            float c4[4];
#pragma unroll
            for (int i = 0; i < 4; ++i) {
                const float send = b8 ? c8[i] : c8[4 + i];
                const float keep = b8 ? c8[4 + i] : c8[i];
                c4[i] = keep + xr(send, 8);
            }
            float c2[2];
#pragma unroll
            for (int i = 0; i < 2; ++i) {
                const float send = b4 ? c4[i] : c4[2 + i];
                const float keep = b4 ? c4[2 + i] : c4[i];
                c2[i] = keep + xr(send, 4);
            }
            const float send1 = b2 ? c2[0] : c2[1];
            const float keep1 = b2 ? c2[1] : c2[0];
            float s = keep1 + xr(send1, 2);
            s = s + xr(s, 1);
            const int head = (lane >> 1) & 15;
            if ((lane & 1) == 0 && head < G) sp[spi(head, c)] = s * scale;
        } else {
#pragma unroll
        for (int h = 0; h < G; ++h) {
            const sycl::float4 qa =
                *reinterpret_cast<const sycl::float4 *>(&sq[h][lane * 8]);
            const sycl::float4 qb =
                *reinterpret_cast<const sycl::float4 *>(&sq[h][lane * 8 + 4]);
            float s = k8[0] * qa.x() + k8[1] * qa.y() + k8[2] * qa.z() +
                      k8[3] * qa.w() + k8[4] * qb.x() + k8[5] * qb.y() +
                      k8[6] * qb.z() + k8[7] * qb.w();
            s = warp_sum(s);
            if (lane == 0) sp[spi(h, c)] = s * scale;
        }
        }
    }
    /*
    DPCT1065: Consider replacing sycl::nd_item::barrier() with
    sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
    performance if there is no access to global memory.
    */
    item_ct1.barrier();
    // per-head chunk max and exp-sum: warp w handles heads w and w+8.
    for (int h = warp; h < G; h += WARPS) {
        const float a = sp[spi(h, lane)], b = sp[spi(h, lane + 32)];
        const float m = warp_max(sycl::fmax(a, b));
        const float ea = (lane < n_here && srow[lane] >= 0)
                             ? sycl::native::exp(a - m)
                             : 0.0f;
        const float eb = (lane + 32 < n_here && srow[lane + 32] >= 0)
                             ? sycl::native::exp(b - m)
                             : 0.0f;
        sp[spi(h, lane)] = ea;
        sp[spi(h, lane + 32)] = eb;
        const float l = warp_sum(ea + eb);
        if (lane == 0) { part_m[slot * G + h] = m; part_l[slot * G + h] = l; }
    }
    /*
    DPCT1065: Consider replacing sycl::nd_item::barrier() with
    sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
    performance if there is no access to global memory.
    */
    item_ct1.barrier();
    // values: thread t owns dimension t for all 12 heads.
    float acc[G];
#pragma unroll
    for (int h = 0; h < G; ++h) acc[h] = 0.0f;
    for (int c = 0; c < n_here; ++c) {
        if (srow[c] < 0) continue;   // masked above, weight 0
        float v;
        if constexpr (KV_MODE == 0) {
            v = sycl::vec<sycl::half, 1>(
                    sycl::bit_cast<sycl::half, unsigned short>(
                        p.v_pool[srow[c] * HD + t]))
                    .convert<float, sycl::rounding_mode::automatic>()[0];
        } else if constexpr (KV_MODE == 1) {
            const float sc =
                sycl::vec<sycl::half, 1>(
                    sycl::bit_cast<sycl::half, unsigned short>(
                        p.v_scale[srow[c] * (HD / KV_Q8_GROUP) +
                                  t / KV_Q8_GROUP]))
                    .convert<float, sycl::rounding_mode::automatic>()[0];
            v = (float) p.v_q[srow[c] * HD + t] * sc;
        } else {   // modes 2 and 3: V is rotated Q4_0 (kv_q4.hpp); the caller rotates the output back
            constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
            const int b = t / QK4_0;
            const int rem = t % QK4_0;
            const block_q4_0* blk = reinterpret_cast<const block_q4_0*>(p.v_q4 + srow[c] * bytes_per_head) + b;
            const float d =
                sycl::vec<sycl::half, 1>(
                    sycl::bit_cast<sycl::half, unsigned short>(blk->d))
                    .convert<float, sycl::rounding_mode::automatic>()[0];
            const int j = rem < 16 ? rem : (rem - 16);
            const uint8_t byte = blk->qs[j];
            const int nibble = (rem < 16) ? ((byte & 0x0F) - 8) : ((byte >> 4) - 8);
            v = (float) nibble * d;
        }
        if constexpr (RS != 0) {
            // the cell's 12 probabilities as three vector loads (the layout is [cell][head]); the same fma per head in the same order
#pragma unroll
            for (int q4 = 0; q4 < G / 4; ++q4) {
                const sycl::float4 pv = *reinterpret_cast<const sycl::float4 *>(&sp[c * G + 4 * q4]);
                acc[4 * q4 + 0] = sycl::fma(pv.x(), v, acc[4 * q4 + 0]);
                acc[4 * q4 + 1] = sycl::fma(pv.y(), v, acc[4 * q4 + 1]);
                acc[4 * q4 + 2] = sycl::fma(pv.z(), v, acc[4 * q4 + 2]);
                acc[4 * q4 + 3] = sycl::fma(pv.w(), v, acc[4 * q4 + 3]);
            }
        } else {
#pragma unroll
        for (int h = 0; h < G; ++h) acc[h] = sycl::fma(sp[spi(h, c)], v, acc[h]);
        }
    }
#pragma unroll
    for (int h = 0; h < G; ++h) part_acc[((size_t) slot * G + h) * HD + t] = acc[h];
}

// PR #540 (sskver): the same attention with fewer shuffles and less shared-memory traffic, bit for bit the kernel
// above (same operands, same order); +7% prompt speed on a V100, where this kernel reads every prompt chunk.  It
// uses more registers (80 vs 38 on sm_70), so it runs on cards below sm_75 only, and exists only in the experimental
// build (-DSTRATA_EXPERIMENTAL_SM60=ON): the ready-made engine keeps exactly the kernel above.
#if defined(STRATA_EXPERIMENTAL_SM60)
// The 12 heads' lane sums (`part[12..15]` zero), reduce-scattered with warp_sum's pairing order (xor 16, 8, 4, 2, 1):
// every output adds the same two operands at every level, so each sum is bit for bit warp_sum's (float add commutes),
// for 16 shuffles instead of 12 x 5.  Lane l ends holding head head_of_lane(l); lanes l and l^1 agree.
__device__ __forceinline__ float reduce12(const float (&part)[16], int lane) {
    float r8[8];
#pragma unroll
    for (int i = 0; i < 8; ++i) {
        const bool hi = (lane & 16) != 0;
        r8[i] = (hi ? part[i + 8] : part[i]) + __shfl_xor_sync(0xffffffffu, hi ? part[i] : part[i + 8], 16);
    }
    float r4[4];
#pragma unroll
    for (int i = 0; i < 4; ++i) {
        const bool hi = (lane & 8) != 0;
        r4[i] = (hi ? r8[i + 4] : r8[i]) + __shfl_xor_sync(0xffffffffu, hi ? r8[i] : r8[i + 4], 8);
    }
    float r2[2];
#pragma unroll
    for (int i = 0; i < 2; ++i) {
        const bool hi = (lane & 4) != 0;
        r2[i] = (hi ? r4[i + 2] : r4[i]) + __shfl_xor_sync(0xffffffffu, hi ? r4[i] : r4[i + 2], 4);
    }
    const bool hi2 = (lane & 2) != 0;
    const float r1 = (hi2 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi2 ? r2[0] : r2[1], 2);
    return r1 + __shfl_xor_sync(0xffffffffu, r1, 1);
}
__device__ __forceinline__ int head_of_lane(int lane) {
    return ((lane >> 4) & 1) * 8 + ((lane >> 3) & 1) * 4 + ((lane >> 2) & 1) * 2 + ((lane >> 1) & 1);
}

// One V element of pool row `row`, dimension `d` (the value loop's own thread index), per KV format.
template <int KV_MODE>
__device__ __forceinline__ float load_v1(const QsaAttnPools& p, long long row, int d) {
    if constexpr (KV_MODE == 0) {
        return __half2float(__ushort_as_half(p.v_pool[row * HD + d]));
    } else if constexpr (KV_MODE == 1) {
        const float sc = __half2float(__ushort_as_half(p.v_scale[row * (HD / KV_Q8_GROUP) + d / KV_Q8_GROUP]));
        return (float) p.v_q[row * HD + d] * sc;
    } else {   // modes 2 and 3: V is rotated Q4_0 (kv_q4.hpp); the caller rotates the output back
        constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
        const int b = d / QK4_0;
        const int rem = d % QK4_0;
        const block_q4_0* blk = reinterpret_cast<const block_q4_0*>(p.v_q4 + row * bytes_per_head) + b;
        const float dd = __half2float(__ushort_as_half(blk->d));
        const int j = rem < 16 ? rem : (rem - 16);
        const uint8_t byte = blk->qs[j];
        const int nibble = (rem < 16) ? ((byte & 0x0F) - 8) : ((byte >> 4) - 8);
        return (float) nibble * dd;
    }
}

template <int KV_MODE>
__global__ void __launch_bounds__(THREADS) attn_chunk_kernel_pre75(const float* __restrict__ q, QsaAttnPools p,
                                                             const int32_t* __restrict__ ids,
                                                             const int32_t* __restrict__ step, int n_kv_heads,
                                                             int page_size, float scale, float* __restrict__ part_acc,
                                                             float* __restrict__ part_m, float* __restrict__ part_l,
                                                             int n_chunks, int cap = 0, long long scratch_stride = 0) {
    // batched form: query blockIdx.z, with its own q row, selection, step and scratch
    q += (size_t) blockIdx.z * (size_t) (n_kv_heads * G) * HD;
    ids += (size_t) blockIdx.z * (size_t) cap;
    step += (size_t) blockIdx.z * kStepCount;
    part_acc += (size_t) blockIdx.z * (size_t) scratch_stride;
    part_m += (size_t) blockIdx.z * (size_t) scratch_stride;
    part_l += (size_t) blockIdx.z * (size_t) scratch_stride;
    __shared__ __align__(16) float sq[G][HD];     // 12 KB: this KV head's query heads (dead once the scores are done)
    __shared__ float sp[G][CHUNK];                // scores
    __shared__ long long srow[CHUNK];             // pool row of each cell (page, kv head, slot)
    // probabilities, cell-major: the value loop reads one cell's 12 as three float4.  They are first written after the
    // __syncthreads that ends the score phase, the last reader of sq, so they share its storage (64 x 12 of its 256 x 12
    // floats) and the block's shared memory stays what it was before the cell-major layout.
    float (*const spt)[G] = reinterpret_cast<float (*)[G]>(&sq[0][0]);
    static_assert(CHUNK * G <= G * HD, "spt must fit in sq");
    const int n_ids = __ldg(step + kStepWidth);
    const int chunk = blockIdx.x, kvh = blockIdx.y;
    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
    const int c0 = chunk * CHUNK;
    const int n_here = min(CHUNK, n_ids - c0);
    const int slot = kvh * n_chunks + chunk;
    if (n_here <= 0) {
        if (t < G) { part_m[slot * G + t] = -FLT_MAX; part_l[slot * G + t] = 0.0f; }
        return;
    }
    for (int i = t; i < G * HD; i += THREADS) sq[i / HD][i % HD] = q[(size_t) (kvh * G) * HD + i];
    if (t < CHUNK) {
        long long r = -1;
        if (t < n_here) {
            const int cell = ids[c0 + t];
            const long long page = (long long) p.page_table[cell / page_size];
            // a block the KV streaming could not make resident keeps page -1 (ctl[3]); its cells are masked
            // (score -FLT_MAX, weight 0) instead of being read from before the pool.
            if (page >= 0) r = (page * n_kv_heads + kvh) * page_size + (cell % page_size);
        }
        srow[t] = r;
    }
    __syncthreads();
    // scores: each warp takes cells warp, warp+8, ...; each lane holds 8 of the 256 dimensions.  NC cells per step
    // (warp + 8 * (i + j)): a lane's q slice comes out of shared memory once for all of them, which cuts the traffic
    // that bounds this phase (24 x 16 B per lane per cell, against the FMAs) by NC; every dot product is the same
    // expression as for a single cell, so the scores are unchanged.
    constexpr int NC = 2;   // 4 measured slower on a V100 (the registers cost occupancy)
    const int hd = head_of_lane(lane);
    for (int i = 0; i < CHUNK / WARPS; i += NC) {
        int cc[NC];
        bool all_ok = true;
#pragma unroll
        for (int j = 0; j < NC; ++j) {
            cc[j] = warp + WARPS * (i + j);
            all_ok = all_ok && cc[j] < n_here && srow[cc[j]] >= 0;
        }
        if (all_ok) {
            float kk[NC][8];
#pragma unroll
            for (int j = 0; j < NC; ++j) load8<KV_MODE>(p, false, srow[cc[j]], lane * 8, kk[j]);
            float pp[NC][16];
#pragma unroll
            for (int h = 0; h < G; ++h) {
                const float4 qa = *reinterpret_cast<const float4*>(&sq[h][lane * 8]);
                const float4 qb = *reinterpret_cast<const float4*>(&sq[h][lane * 8 + 4]);
#pragma unroll
                for (int j = 0; j < NC; ++j)
                    pp[j][h] = kk[j][0] * qa.x + kk[j][1] * qa.y + kk[j][2] * qa.z + kk[j][3] * qa.w +
                               kk[j][4] * qb.x + kk[j][5] * qb.y + kk[j][6] * qb.z + kk[j][7] * qb.w;
            }
#pragma unroll
            for (int j = 0; j < NC; ++j) {
#pragma unroll
                for (int h = G; h < 16; ++h) pp[j][h] = 0.0f;
                const float sj = reduce12(pp[j], lane);
                if ((lane & 1) == 0 && hd < G) sp[hd][cc[j]] = sj * scale;
            }
        } else {
            for (int k = 0; k < NC; ++k) {   // a masked or out-of-range cell in the group: one cell at a time
                const int c = cc[k];
                if (c >= n_here || srow[c] < 0) {
                    if (lane < G) sp[lane][c] = -FLT_MAX;
                    continue;
                }
                float k8[8];
                load8<KV_MODE>(p, false, srow[c], lane * 8, k8);
                float part[16];
#pragma unroll
                for (int h = 0; h < G; ++h) {
                    const float4 qa = *reinterpret_cast<const float4*>(&sq[h][lane * 8]);
                    const float4 qb = *reinterpret_cast<const float4*>(&sq[h][lane * 8 + 4]);
                    part[h] = k8[0] * qa.x + k8[1] * qa.y + k8[2] * qa.z + k8[3] * qa.w +
                              k8[4] * qb.x + k8[5] * qb.y + k8[6] * qb.z + k8[7] * qb.w;
                }
#pragma unroll
                for (int h = G; h < 16; ++h) part[h] = 0.0f;
                const float s = reduce12(part, lane);
                if ((lane & 1) == 0 && hd < G) sp[hd][c] = s * scale;
            }
        }
    }
    __syncthreads();
    // per-head chunk max and exp-sum: warp w handles heads w and w+8.
    for (int h = warp; h < G; h += WARPS) {
        const float a = sp[h][lane], b = sp[h][lane + 32];
        const float m = warp_max(fmaxf(a, b));
        const float ea = (lane < n_here && srow[lane] >= 0) ? __expf(a - m) : 0.0f;
        const float eb = (lane + 32 < n_here && srow[lane + 32] >= 0) ? __expf(b - m) : 0.0f;
        spt[lane][h] = ea;
        spt[lane + 32][h] = eb;
        const float l = warp_sum(ea + eb);
        if (lane == 0) { part_m[slot * G + h] = m; part_l[slot * G + h] = l; }
    }
    __syncthreads();
    // values: thread t owns dimension t for all 12 heads.
    float acc[G];
#pragma unroll
    for (int h = 0; h < G; ++h) acc[h] = 0.0f;
    // One cell's 12 weights (three broadcast float4 loads) into the 12 accumulators; the cells are folded in
    // ascending order, so the sums are the plain loop's, bit for bit.
    const auto fold = [&](int c, float v) {
        const float4* w4 = reinterpret_cast<const float4*>(spt[c]);
        const float4 w0 = w4[0], w1 = w4[1], w2 = w4[2];
        acc[0] = fmaf(w0.x, v, acc[0]);  acc[1] = fmaf(w0.y, v, acc[1]);
        acc[2] = fmaf(w0.z, v, acc[2]);  acc[3] = fmaf(w0.w, v, acc[3]);
        acc[4] = fmaf(w1.x, v, acc[4]);  acc[5] = fmaf(w1.y, v, acc[5]);
        acc[6] = fmaf(w1.z, v, acc[6]);  acc[7] = fmaf(w1.w, v, acc[7]);
        acc[8] = fmaf(w2.x, v, acc[8]);  acc[9] = fmaf(w2.y, v, acc[9]);
        acc[10] = fmaf(w2.z, v, acc[10]); acc[11] = fmaf(w2.w, v, acc[11]);
    };
    int c = 0;
    // four cells at a time with their V loads issued together (the loop was one dependent load chain per cell);
    // a group holding a masked cell (page -1, rare) leaves this loop and the scalar one below finishes the chunk.
    for (; c + 4 <= n_here; c += 4) {
        const long long r0 = srow[c], r1 = srow[c + 1], r2 = srow[c + 2], r3 = srow[c + 3];
        if ((r0 | r1 | r2 | r3) < 0) break;
        const float v0 = load_v1<KV_MODE>(p, r0, t), v1 = load_v1<KV_MODE>(p, r1, t);
        const float v2 = load_v1<KV_MODE>(p, r2, t), v3 = load_v1<KV_MODE>(p, r3, t);
        fold(c, v0);
        fold(c + 1, v1);
        fold(c + 2, v2);
        fold(c + 3, v3);
    }
    for (; c < n_here; ++c) {
        if (srow[c] < 0) continue;   // masked above, weight 0
        fold(c, load_v1<KV_MODE>(p, srow[c], t));
    }
#pragma unroll
    for (int h = 0; h < G; ++h) part_acc[((size_t) slot * G + h) * HD + t] = acc[h];
}
#endif  // STRATA_EXPERIMENTAL_SM60

__dpct_inline__ void attn_merge_kernel(const float *__restrict__ part_acc,
                                       const float *__restrict__ part_m,
                                       const float *__restrict__ part_l,
                                       int n_chunks, float *__restrict__ attn,
                                       long long scratch_stride = 0) {
    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
    part_acc += (size_t)item_ct1.get_group(1) * (size_t)scratch_stride;
    part_m += (size_t)item_ct1.get_group(1) * (size_t)scratch_stride;
    part_l += (size_t)item_ct1.get_group(1) * (size_t)scratch_stride;
    attn += (size_t)item_ct1.get_group(1) *
            (size_t)item_ct1.get_group_range(2) * HD;
    const int h = item_ct1.get_group(2); // global query head
    const int kvh = h / G, hl = h % G;
    const int d = item_ct1.get_local_id(2);
    float M = -FLT_MAX;
#pragma unroll
    for (int c = 0; c < n_chunks; ++c)
        M = sycl::fmax(M, part_m[(kvh * n_chunks + c) * G + hl]);
    float L = 0.0f, acc = 0.0f;
    for (int c = 0; c < n_chunks; ++c) {
        const int slot = kvh * n_chunks + c;
        const float m = part_m[slot * G + hl];
        if (m == -FLT_MAX) continue;
        const float w = sycl::native::exp(m - M);
        L = sycl::fma((float)(part_l[slot * G + hl]), (float)w, L);
        acc = sycl::fma((float)(part_acc[((size_t)slot * G + hl) * HD + d]),
                        (float)w, acc);
    }
    attn[(size_t) h * HD + d] = L > 0.0f ? acc / L : 0.0f;
}

#if defined(STRATA_EXPERIMENTAL_SM60)
// the current device is below sm_75 (per device: a layer split can mix cards); STRATA_ATTN_PRE75=0 turns PR #540's
// kernel off (A/B)
bool pre75_attn() {
    static const bool off = [] {
        const char* e = std::getenv("STRATA_ATTN_PRE75");
        return e != nullptr && e[0] == '0';
    }();
    static int cc[64] = {};
    int dev = 0;
    if (off) return false;
    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
    if (cc[dev] == 0) {
        int major = 0, minor = 0;
        if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess ||
            cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev) != cudaSuccess) {
            cudaGetLastError();
            return false;
        }
        cc[dev] = 10 * major + minor;
    }
    return cc[dev] < 75;
}
#define STRATA_ATTN_CHUNK(M) (pre75_attn() ? attn_chunk_kernel_pre75<M> : attn_chunk_kernel<M>)
#else
#define STRATA_ATTN_CHUNK(M) attn_chunk_kernel<M>
#endif

}  // namespace

void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
                           int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {
    if (n_q <= 0) return;
    if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps ||
        !pools.page_table || n_q > 65535) {
        std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");
        std::exit(1);
    }
    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
                        : (pools.k_q != nullptr ? 1 : 0));
    const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
    // per query: [acc: n_chunks*n_head*HD][m: n_chunks*n_head][l: n_chunks*n_head], all offsets from one stride
    const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
    float* part_acc = scratch;
    float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;
    float* part_l = part_m + (size_t) n_chunks * s.n_head;
    const float scale = 1.0f / sqrtf((float) HD);
    const dpct::dim3 grid((unsigned)n_chunks, (unsigned)s.n_head_kv,
                          (unsigned)n_q);
    dpct::queue_ptr st = strata::q_of(stream);
    // S25: the score phase one cell per thread (bit-identical scores, no shuffle reductions).  Prompt batches take
    // it by default: on 2x Arc Pro B65 it reads a 4,106-token prompt 5.2% faster, 8,191 tokens 5.2% and 16,380
    // tokens 4.8%, with the same tokens; the decode window's few queries per call are 1.4% slower with it (56.6-57.0
    // against 55.9-56.3 tok/s), so the choice follows the query count.  It is a Battlemage (Xe2) default only: on an Arc
    // A750 (Alchemist) the same switch reads a 4,106-token prompt 0.7% slower (three interleaved runs, 4 of 5 pairs
    // faster with it off), so the generation comes from the device's architecture id (strata::intel_gpu_gen: the
    // runtime's architecture enum first, the name only as a fallback), never from the name alone.
    // STRATA_ATTN_LANECELL=1/0 forces either way on any device.
    static const int lane_cell_env = [] {
        const char* v = std::getenv("STRATA_ATTN_LANECELL");
        return v != nullptr && v[0] != '\0' ? std::atoi(v) : -1;
    }();
    const bool lane_cell = lane_cell_env >= 0
        ? lane_cell_env != 0
        : (n_q >= 16 && strata::intel_gpu_gen(st->get_device()) == strata::IntelGpuGen::Battlemage);
#define STRATA_ATTN_LC(M)                                                      \
    {                                                                          \
        auto exp_props = sycl::ext::oneapi::experimental::properties{          \
            sycl::ext::oneapi::experimental::use_root_sync};                   \
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});  \
                                                                               \
        st->submit([&](sycl::handler &cgh) {                                   \
            auto q_ct0 = q;                                                    \
            QsaAttnPools pools_ct1 = pools;                                    \
            auto ids_ct2 = ids;                                                \
            auto steps_ct3 = steps;                                            \
            auto s_n_head_kv_ct4 = (int)s.n_head_kv;                           \
            auto s_page_size_ct5 = (int)s.page_size;                           \
            auto scale_ct6 = scale;                                            \
            auto part_acc_ct7 = part_acc;                                      \
            auto part_m_ct8 = part_m;                                          \
            auto part_l_ct9 = part_l;                                          \
            auto n_chunks_ct10 = n_chunks;                                     \
            auto cap_ct11 = (int)cap;                                          \
            auto stride_ct12 = stride;                                         \
                                                                               \
            cgh.parallel_for<dpct_kernel_name<class attn_chunk_kernel_6e87d5,  \
                                              dpct_kernel_scalar<M>,           \
                                              dpct_kernel_scalar<true>>>(      \
                sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),           \
                                  sycl::range(1, 1, THREADS)),                 \
                exp_props,                                                     \
                [=](sycl::nd_item<3> item_ct1)                                 \
                    [[sycl::reqd_sub_group_size(32)]] {                        \
                        attn_chunk_kernel<M, true>(                            \
                            q_ct0, pools_ct1, ids_ct2, steps_ct3,              \
                            s_n_head_kv_ct4, s_page_size_ct5, scale_ct6,       \
                            part_acc_ct7, part_m_ct8, part_l_ct9,              \
                            n_chunks_ct10, cap_ct11, stride_ct12);             \
                    });                                                        \
        });                                                                    \
    }
#define STRATA_ATTN_RSK(M, V)                                                      \
    {                                                                          \
        auto exp_props = sycl::ext::oneapi::experimental::properties{          \
            sycl::ext::oneapi::experimental::use_root_sync};                   \
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});  \
                                                                               \
        st->submit([&](sycl::handler &cgh) {                                   \
            auto q_ct0 = q;                                                    \
            QsaAttnPools pools_ct1 = pools;                                    \
            auto ids_ct2 = ids;                                                \
            auto steps_ct3 = steps;                                            \
            auto s_n_head_kv_ct4 = (int)s.n_head_kv;                           \
            auto s_page_size_ct5 = (int)s.page_size;                           \
            auto scale_ct6 = scale;                                            \
            auto part_acc_ct7 = part_acc;                                      \
            auto part_m_ct8 = part_m;                                          \
            auto part_l_ct9 = part_l;                                          \
            auto n_chunks_ct10 = n_chunks;                                     \
            auto cap_ct11 = (int)cap;                                          \
            auto stride_ct12 = stride;                                         \
                                                                               \
            cgh.parallel_for<dpct_kernel_name<class attn_chunk_kernel_rs_7c2a91,  \
                                              dpct_kernel_scalar<M>,           \
                                              dpct_kernel_scalar<V>>>(      \
                sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),           \
                                  sycl::range(1, 1, THREADS)),                 \
                exp_props,                                                     \
                [=](sycl::nd_item<3> item_ct1)                                 \
                    [[sycl::reqd_sub_group_size(32)]] {                        \
                        attn_chunk_kernel<M, false, V>(                            \
                            q_ct0, pools_ct1, ids_ct2, steps_ct3,              \
                            s_n_head_kv_ct4, s_page_size_ct5, scale_ct6,       \
                            part_acc_ct7, part_m_ct8, part_l_ct9,              \
                            n_chunks_ct10, cap_ct11, stride_ct12);             \
                    });                                                        \
        });                                                                    \
    }
#undef STRATA_ATTN_RSK_DEFINED
    // 151-b70: STRATA_ATTN_RS=2 (default): the probabilities in [cell][head] order and the value phase reading a cell's 12 as three vectors - the same
    // bits (first-token logits md5 equal at 4K and 16K), +0.8..0.9% prompt tok/s on the B70; =1 adds a 16-shuffle reduce-scatter for the scores
    // (bit-exact too, but -1.5% here: the score phase is not shuffle bound); =0 the layout before
    static const int rs_on = [] { const char* v = std::getenv("STRATA_ATTN_RS"); return v ? std::atoi(v) : 2; }();
    if (lane_cell) {
        if (kv_mode == 3) { STRATA_ATTN_LC(3); }
        else if (kv_mode == 2) { STRATA_ATTN_LC(2); }
        else if (kv_mode == 1) { STRATA_ATTN_LC(1); }
        else { STRATA_ATTN_LC(0); }
    } else if (rs_on == 1) {
        if (kv_mode == 3) { STRATA_ATTN_RSK(3, 1); }
        else if (kv_mode == 2) { STRATA_ATTN_RSK(2, 1); }
        else if (kv_mode == 1) { STRATA_ATTN_RSK(1, 1); }
        else { STRATA_ATTN_RSK(0, 1); }
    } else if (rs_on == 2) {
        if (kv_mode == 3) { STRATA_ATTN_RSK(3, 2); }
        else if (kv_mode == 2) { STRATA_ATTN_RSK(2, 2); }
        else if (kv_mode == 1) { STRATA_ATTN_RSK(1, 2); }
        else { STRATA_ATTN_RSK(0, 2); }
    } else
#undef STRATA_ATTN_LC
#undef STRATA_ATTN_RSK
    if (kv_mode == 3)
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_dfacfc,
                                          dpct_kernel_scalar<3>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<3>(q, pools, ids, steps, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, (int)cap, stride);
            });
    } else if (kv_mode == 2)
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_b0d448,
                                          dpct_kernel_scalar<2>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<2>(q, pools, ids, steps, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, (int)cap, stride);
            });
    } else if (kv_mode == 1)
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_2ad0a4,
                                          dpct_kernel_scalar<1>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<1>(q, pools, ids, steps, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, (int)cap, stride);
            });
    } else {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_5c04f7,
                                          dpct_kernel_scalar<0>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<0>(q, pools, ids, steps, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, (int)cap, stride);
            });
    }
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        st->parallel_for<dpct_kernel_name<class attn_merge_kernel_338175>>(
            sycl::nd_range<3>(
                sycl::range(1, (unsigned)n_q, (unsigned)s.n_head) *
                    sycl::range(1, 1, HD),
                sycl::range(1, 1, HD)),
            exp_props, [=](sycl::nd_item<3> item_ct1) {
                attn_merge_kernel(part_acc, part_m, part_l, n_chunks, attn,
                                  stride);
            });
    }
    /*
    DPCT1010: SYCL uses exceptions to report errors and does not use the
    error codes. The cudaGetLastError function call was replaced with 0. You
    need to rewrite this code.
    */
    const dpct::err0 e = 0;
}

uint64_t qsa_decode_attn_scratch_floats(int64_t cap, const QsaShapes& s) {
    const int64_t chunks = (cap + CHUNK - 1) / CHUNK;
    return (uint64_t) chunks * (uint64_t) s.n_head * (HD + 2) + 64;
}

void qsa_decode_attn_step(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* step,
                          int64_t cap, const QsaShapes& s, float* scratch, float* attn, void* stream) {
    if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !step ||
        !pools.page_table) {
        std::fprintf(stderr, "qsa_decode_attn: unsupported geometry or missing buffers\n");
        std::exit(1);
    }
    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
                        : (pools.k_q != nullptr ? 1 : 0));
    if (kv_mode == 3 ? (!pools.k_scale || !pools.v_q4)
                     : (kv_mode == 2 ? (!pools.v_q4) : (kv_mode == 1 ? (!pools.v_q || !pools.k_scale || !pools.v_scale)
                                                                     : (!pools.k_pool || !pools.v_pool)))) {
        std::fprintf(stderr, "qsa_decode_attn: incomplete KV pools\n");
        std::exit(1);
    }
    const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
    float* part_acc = scratch;
    float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;
    float* part_l = part_m + (size_t) n_chunks * s.n_head;
    const float scale = 1.0f / sqrtf((float) HD);
    const dpct::dim3 grid((unsigned)n_chunks, (unsigned)s.n_head_kv);
    dpct::queue_ptr st = strata::q_of(stream);
    if (kv_mode == 3)
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_f262e6,
                                          dpct_kernel_scalar<3>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<3>(q, pools, ids, step, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, 0, 0);
            });
    } else if (kv_mode == 2)
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_4fd923,
                                          dpct_kernel_scalar<2>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<2>(q, pools, ids, step, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, 0, 0);
            });
    } else if (kv_mode == 1)
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_f6855e,
                                          dpct_kernel_scalar<1>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<1>(q, pools, ids, step, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, 0, 0);
            });
    } else {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};
        dpct::has_capability_or_fail(st->get_device(), {sycl::aspect::fp16});

        st->parallel_for<dpct_kernel_name<class attn_chunk_kernel_7c8e09,
                                          dpct_kernel_scalar<0>>>(
            sycl::nd_range<3>(grid * sycl::range(1, 1, THREADS),
                              sycl::range(1, 1, THREADS)),
            exp_props,
            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
                attn_chunk_kernel<0>(q, pools, ids, step, (int)s.n_head_kv,
                                     (int)s.page_size, scale, part_acc, part_m,
                                     part_l, n_chunks, 0, 0);
            });
    }
    {
        auto exp_props = sycl::ext::oneapi::experimental::properties{
            sycl::ext::oneapi::experimental::use_root_sync};

        st->parallel_for<dpct_kernel_name<class attn_merge_kernel_c055e7>>(
            sycl::nd_range<3>(sycl::range(1, 1, (unsigned)s.n_head) *
                                  sycl::range(1, 1, HD),
                              sycl::range(1, 1, HD)),
            exp_props, [=](sycl::nd_item<3> item_ct1) {
                attn_merge_kernel(part_acc, part_m, part_l, n_chunks, attn, 0);
            });
    }
    /*
    DPCT1010: SYCL uses exceptions to report errors and does not use the
    error codes. The cudaGetLastError function call was replaced with 0. You
    need to rewrite this code.
    */
    const dpct::err0 e = 0;
}

}  // namespace strata::kernels