Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// Adapted from llama.cpp 3cf03257f219afbe7334045ff7c6a06ac68c627d:
// ggml/src/ggml-cuda/{norm.cu,common.cuh,unary.cu}.
//
// MIT License
// Copyright (c) 2023-2026 The ggml authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include <dpct/dpct.hpp>
#include "strata/sycl_queue.hpp"
#include "strata/kernels/native_qsa.hpp"
#include <atomic>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <stdexcept>
#include <string>
namespace strata::kernels {
namespace {
std::atomic<bool> enabled{false};
__dpct_inline__ float warp_sum(float value) {
#pragma unroll
for (int offset = 16; offset; offset >>= 1)
/*
DPCT1108: '__shfl_xor_sync' was migrated with the experimental
feature masked sub_group function which may not be supported by all
compilers or runtimes. You may need to adjust the code.
*/
value += dpct::experimental::permute_sub_group_by_xor(
0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(),
value, offset);
return value;
}
template <int BlockSize>
__dpct_inline__ void norm(const float *input, const float *__restrict__ gamma,
float *output, int n_cols, float epsilon) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int tid = item_ct1.get_local_id(2);
const std::size_t row_offset = std::size_t(item_ct1.get_group(2)) * n_cols;
input += row_offset; output += row_offset;
float partial = 0.0f;
for (std::size_t col = tid; col < std::size_t(n_cols); col += BlockSize) {
const float x = input[col];
partial += x * x;
}
auto &sums =
*sycl::ext::oneapi::group_local_memory_for_overwrite<float[32]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
partial = warp_sum(partial);
const int lane = tid % 32;
if (lane == 0) sums[tid / 32] = partial;
// All reads of input for the reduction precede this barrier. Afterwards,
// each thread reads/writes only its own elements, allowing exact in-place use.
item_ct1.barrier(sycl::access::fence_space::local_space);
partial = lane < BlockSize / 32 ? sums[lane] : 0.0f;
partial = warp_sum(partial);
const float mean = partial / n_cols;
const float scale = sycl::rsqrt(mean + epsilon);
#pragma unroll
for (std::size_t col = tid; col < std::size_t(n_cols); col += BlockSize)
output[col] = scale * input[col] * gamma[col];
}
// As norm, with gamma[groups * n_cols]: row r is group r % groups and reads its own slice of gamma.
template <int BlockSize>
__dpct_inline__ void
norm_grouped(const float *input, const float *__restrict__ gamma, float *output,
int n_cols, int groups, float epsilon) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int tid = item_ct1.get_local_id(2);
const std::size_t row_offset = std::size_t(item_ct1.get_group(2)) * n_cols;
input += row_offset; output += row_offset;
gamma += std::size_t(item_ct1.get_group(2) % unsigned(groups)) * n_cols;
float partial = 0.0f;
for (std::size_t col = tid; col < std::size_t(n_cols); col += BlockSize) {
const float x = input[col];
partial += x * x;
}
auto &sums =
*sycl::ext::oneapi::group_local_memory_for_overwrite<float[32]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
partial = warp_sum(partial);
const int lane = tid % 32;
if (lane == 0) sums[tid / 32] = partial;
item_ct1.barrier(sycl::access::fence_space::local_space);
partial = lane < BlockSize / 32 ? sums[lane] : 0.0f;
partial = warp_sum(partial);
const float mean = partial / n_cols;
const float scale = sycl::rsqrt(mean + epsilon);
#pragma unroll
for (std::size_t col = tid; col < std::size_t(n_cols); col += BlockSize)
output[col] = scale * input[col] * gamma[col];
}
__dpct_inline__ void gate(const float *attn, const float *__restrict__ q_full,
float *output, int n_head, int head_dim) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const std::size_t i =
std::size_t(item_ct1.get_group(2)) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
if (i >= std::size_t(n_head) * head_dim) return;
const std::size_t head = i / head_dim, channel = i % head_dim;
const float raw = q_full[head * 2 * head_dim + head_dim + channel];
const float sigmoid = 1.0f / (1.0f + sycl::native::exp(-raw));
output[i] = attn[i] * sigmoid;
}
std::size_t elements(int cols, int rows) {
if (cols <= 0 || rows <= 0 || std::uint64_t(cols) * rows > std::uint64_t(std::numeric_limits<int>::max()))
throw std::invalid_argument("native QSA requires positive bounded dimensions");
return std::size_t(cols) * rows;
}
bool valid(const void* ptr, std::size_t bytes) {
const auto address = reinterpret_cast<std::uintptr_t>(ptr);
return ptr && address % 4 == 0 && bytes <= UINTPTR_MAX - address;
}
bool overlap(const void* a, std::size_t an, const void* b, std::size_t bn) {
const auto ap = reinterpret_cast<std::uintptr_t>(a), bp = reinterpret_cast<std::uintptr_t>(b);
return ap < bp + bn && bp < ap + an;
}
void buffers(const float* input, std::size_t in_bytes, const float* weight, std::size_t weight_bytes,
float* output, void* stream) {
if (!stream || !valid(input, in_bytes) || !valid(weight, weight_bytes) || !valid(output, in_bytes) ||
overlap(input, in_bytes, weight, weight_bytes) || overlap(output, in_bytes, weight, weight_bytes) ||
(input != output && overlap(input, in_bytes, output, in_bytes)))
throw std::invalid_argument("native QSA requires a stream, aligned spans, and disjoint buffers or exact input/output alias");
}
void check_launch() {
/*
DPCT1010: SYCL uses exceptions to report errors and does not use the
error codes. The cudaGetLastError function call was replaced with 0. You
need to rewrite this code.
*/
const auto result = 0;
/*
DPCT1000: Error handling if-stmt was detected but could not be
rewritten.
*/
if (result != 0)
/*
DPCT1009: SYCL reports errors using exceptions and does not use
error codes. Please replace the "get_error_string_dummy(...)" with a
real error-handling function.
*/
/*
DPCT1001: The statement could not be removed.
*/
throw std::runtime_error(std::string("native QSA launch: ") +
dpct::get_error_string_dummy(result));
}
} // namespace
void native_qsa_set_enabled(bool value) { enabled.store(value, std::memory_order_relaxed); }
bool native_qsa_enabled() { return enabled.load(std::memory_order_relaxed); }
void native_qsa_rms_norm_weighted(const float* input, const float* gamma, float* output,
int n_cols, int n_rows, float epsilon, void* stream) {
const auto count = elements(n_cols, n_rows);
if (!std::isfinite(epsilon) || epsilon < 0.0f)
throw std::invalid_argument("native QSA requires finite nonnegative epsilon");
buffers(input, count * 4, gamma, std::size_t(n_cols) * 4, output, stream);
if (n_cols < 1024)
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
((sycl::queue *)(strata::q_of(stream)))
->parallel_for<
dpct_kernel_name<class norm_5d08bf, dpct_kernel_scalar<256>>>(
sycl::nd_range<3>(sycl::range(1, 1, unsigned(n_rows)) *
sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
norm<256>(input, gamma, output, n_cols, epsilon);
});
} else
/*
DPCT1049: The work-group size passed to the SYCL kernel may exceed
the limit. To get the device limit, query
info::device::max_work_group_size. Adjust the work-group size if needed.
*/
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
((sycl::queue *)(strata::q_of(stream)))
->parallel_for<
dpct_kernel_name<class norm_127936, dpct_kernel_scalar<1024>>>(
sycl::nd_range<3>(sycl::range(1, 1, unsigned(n_rows)) *
sycl::range(1, 1, 1024),
sycl::range(1, 1, 1024)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
norm<1024>(input, gamma, output, n_cols, epsilon);
});
}
check_launch();
}
void native_qsa_rms_norm_grouped(const float* input, const float* gamma, float* output,
int n_cols, int groups, int n_rows, float epsilon, void* stream) {
if (groups < 1) throw std::invalid_argument("native QSA requires positive bounded dimensions");
const auto count = elements(n_cols, n_rows);
if (!std::isfinite(epsilon) || epsilon < 0.0f)
throw std::invalid_argument("native QSA requires finite nonnegative epsilon");
buffers(input, count * 4, gamma, std::size_t(n_cols) * groups * 4, output, stream);
if (n_cols < 1024)
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
((sycl::queue *)(strata::q_of(stream)))
->parallel_for<dpct_kernel_name<class norm_grouped_b5907a,
dpct_kernel_scalar<256>>>(
sycl::nd_range<3>(sycl::range(1, 1, unsigned(n_rows)) *
sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
norm_grouped<256>(input, gamma, output, n_cols, groups,
epsilon);
});
} else
/*
DPCT1049: The work-group size passed to the SYCL kernel may exceed
the limit. To get the device limit, query
info::device::max_work_group_size. Adjust the work-group size if needed.
*/
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
((sycl::queue *)(strata::q_of(stream)))
->parallel_for<dpct_kernel_name<class norm_grouped_927e80,
dpct_kernel_scalar<1024>>>(
sycl::nd_range<3>(sycl::range(1, 1, unsigned(n_rows)) *
sycl::range(1, 1, 1024),
sycl::range(1, 1, 1024)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
norm_grouped<1024>(input, gamma, output, n_cols, groups,
epsilon);
});
}
check_launch();
}
void native_qsa_gate_apply(const float* attn, const float* q_full, float* output,
int n_head, int head_dim, void* stream) {
const auto count = elements(head_dim, n_head);
buffers(attn, count * 4, q_full, count * 8, output, stream);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
((sycl::queue *)(strata::q_of(stream)))
->parallel_for<dpct_kernel_name<class gate_148967>>(
sycl::nd_range<3>(
sycl::range(1, 1, unsigned((count + 255) / 256)) *
sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
gate(attn, q_full, output, n_head, head_dim);
});
}
check_launch();
}
} // namespace strata::kernels