Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

SYCL: trellis experts (TQ2_T, TQK6, TQK7) for decode and the prompt path, Hadamard-folded inputs; tq_sycl_parity (host replay of the kernels bit for bit, dequantizers, Hadamard)

Niko122102791a6d18fb
diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt
index b5eb3f6b5..4dc7ae000 100644
--- a/sycl/CMakeLists.txt
+++ b/sycl/CMakeLists.txt
@@ -424,6 +424,11 @@ if(STRATA_SYCL_PARITY)
add_executable(native_expert_parity ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/native_expert_parity.cpp)
target_include_directories(native_expert_parity PRIVATE ${STRATA_ROOT}/third_party/ggml)
target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu)
+ # Gyro (APR) trellis experts on real GGUF tensors: tq_sycl_parity <pack> <model.gguf> [layer ...]
+ add_executable(tq_sycl_parity ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/tq_sycl_parity.cpp)
+ target_include_directories(tq_sycl_parity PRIVATE ${STRATA_ROOT}/third_party/ggml)
+ target_compile_options(tq_sycl_parity PRIVATE -ffp-contract=off)
+ target_link_libraries(tq_sycl_parity PRIVATE strata_engine strata_kernels strata_kernels_cpu)
# MODEL-UPDATE: the CPU pool's expert rows per format against the RAM bandwidth (as the top-level build's)
add_executable(kq_cpu_bench ${STRATA_ROOT}/src/kernels/cpu/kq_cpu_bench.cpp)
target_link_libraries(kq_cpu_bench PRIVATE strata_kernels_cpu)
diff --git a/sycl/include/strata/core/verify.hpp b/sycl/include/strata/core/verify.hpp
index 894f7b7b7..b38249f4d 100644
--- a/sycl/include/strata/core/verify.hpp
+++ b/sycl/include/strata/core/verify.hpp
@@ -460,6 +460,10 @@ private:
static constexpr int64_t kPcieGroupRows = 4; // the PCIe call's groups side by side (of <= 16)
uint8_t* hit_xq_ = nullptr;
uint8_t* nat_xq_ = nullptr; // plan v0.3 P6: q8_1 activations for a native pack's grouped experts
+ float* nat_xr_ = nullptr; // APR: the Hadamard-rotated expert input (the shared expert and the CPU keep xm); nullptr
+ // unless some layer's gate/up is folded
+ const float* had_sx_ = nullptr; // APR: device sign vectors for x (n_embd) and h (n_ff); nullptr = identity
+ const float* had_sh_ = nullptr;
float* hit_xs_ = nullptr;
void* hit_scratch_ = nullptr;
float *head_mixed_ = nullptr, *head_inj_ = nullptr, *head_logits_ = nullptr;
diff --git a/sycl/include/strata/kernels/tq_lut_i8_sycl.hpp b/sycl/include/strata/kernels/tq_lut_i8_sycl.hpp
new file mode 100644
index 000000000..b63713e82
--- /dev/null
+++ b/sycl/include/strata/kernels/tq_lut_i8_sycl.hpp
@@ -0,0 +1,10 @@
+// The int8 copy of the trellis codebook (src/kernels/cuda/tq_lut_i8.cuh, unchanged) for the SYCL kernels: a const
+// table at namespace scope, which a SYCL kernel reads like ggml's i-quant grids. The .cuh marks it __device__ for
+// CUDA/HIP; here that qualifier is empty for the length of the include.
+#pragma once
+#ifdef __device__
+#error "tq_lut_i8_sycl.hpp: __device__ is already defined"
+#endif
+#define __device__
+#include "../../../../src/kernels/cuda/tq_lut_i8.cuh"
+#undef __device__
diff --git a/sycl/src/core/verify.cpp b/sycl/src/core/verify.cpp
index 1d7501f11..a20208624 100644
--- a/sycl/src/core/verify.cpp
+++ b/sycl/src/core/verify.cpp
@@ -15,6 +15,7 @@
#include "strata/core/peer_experts.hpp"
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
#include "strata/kernels/bf16_gemv.hpp"
#include "strata/kernels/native_router.hpp"
#include "strata/kernels/native_moe.hpp"
@@ -562,6 +563,13 @@ bool Verifier::init(const WeightTable &wt, const ModelGeometry &g,
sink_.ctx = this;
}
+ // APR: the rotated copy of the expert input (nat_xr_, T x N floats) only where a layer's gate/up is
+ // Hadamard-folded; its one reader is that layer's f.had_gu branch in the expert-input step below
+ bool had_gu_any = false;
+ {
+ const auto& lay = strata::kernels::cpu::expert_layout();
+ for (size_t l = 0; lay.native && l < lay.fmt.size(); ++l) had_gu_any = had_gu_any || lay.fmt[l].had_gu;
+ }
// ---- the device arena: the same sequence counted, then carved
auto carve = [&](Bump& b) {
tok_ = b.take<int32_t>(T); step_ = b.take<int32_t>(T * strata::kernels::kStepCount);
@@ -589,6 +597,7 @@ bool Verifier::init(const WeightTable &wt, const ModelGeometry &g,
staging_ = b.take<uint8_t>((uint64_t) kStagingBlobs * strata::kernels::cpu::expert_layout().max_blob);
hit_xq_ = b.take<uint8_t>(T * (N / 32) * 34); hit_xs_ = b.take<float>(T * (N / 32));
nat_xq_ = b.take<uint8_t>(T * (N / 32) * 36);
+ nat_xr_ = had_gu_any ? b.take<float>(T * N) : nullptr;
hit_scratch_ = b.take<uint8_t>(std::max<uint64_t>(
strata::kernels::moe_hit_grouped_scratch_bytes((int64_t) (T * K), g.n_embd, g.n_ff),
strata::kernels::native_expert_scratch_bytes((int64_t) (T * K), g.n_ff)));
@@ -652,6 +661,10 @@ bool Verifier::init(const WeightTable &wt, const ModelGeometry &g,
return false;
}
}
+ { // APR (prism.hadamard, hadamard.txt): the sign vectors on this device, before any graph capture
+ had_sx_ = strata::kernels::hadamard_device_signs(strata::kernels::cpu::hadamard_signs((int) g.n_embd), (int) g.n_embd);
+ had_sh_ = strata::kernels::hadamard_device_signs(strata::kernels::cpu::hadamard_signs((int) g.n_ff), (int) g.n_ff);
+ }
sink_.staging = (unsigned long long) staging_;
sink_.staging_cap = kStagingBlobs;
(void) TS;
@@ -1395,8 +1408,17 @@ bool Verifier::record_window(int T, dpct::queue_ptr cs, std::string &err) {
if (sh_fork) dpct::sync_barrier(ev_join_, sh_cs_);
}
if (strata::kernels::cpu::expert_layout().native) {
- if (!qdedup)
- if (!q8_ffn) quantize_q8_1_rows(xm, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);
+ // APR: Hadamard-folded gate/up take H (s * x); xm itself stays as it is (the shared expert above and the
+ // doorbell copy for the CPU pool, which rotates its own in native_quant_act). The q8_1 image of xm that
+ // STRATA_QFUSE / STRATA_VERIFY_QDEDUP may already have written is of the unrotated input, so a folded layer
+ // always rewrites it from the rotated rows (same stream, after the shared expert has read it).
+ const auto& f = strata::kernels::cpu::expert_layout().fmt[(size_t) l];
+ if (f.had_gu) { // nat_xr_ exists: init carved it because this layer is folded (had_gu_any)
+ hadamard_rows(xm, nat_xr_ + (size_t) tb * N, n, N, f.had_block, had_sx_, cs);
+ quantize_q8_1_rows(nat_xr_ + (size_t) tb * N, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);
+ } else if (!qdedup && !q8_ffn) {
+ quantize_q8_1_rows(xm, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);
+ }
} else
quantize_q8_0_scaled(xm, hit_xq_ + (size_t) tb * (N / 32) * 34, hit_xs_ + (size_t) tb * (N / 32), (int64_t) n * N, cs);
stamp(l, 18, grp);
@@ -1432,7 +1454,8 @@ bool Verifier::record_window(int T, dpct::queue_ptr cs, std::string &err) {
if (lay.native) {
// the layer's GGUF formats (i-quant gate/up, Q2_0 / IQ4_NL down)
const auto& f = lay.fmt[(size_t) l];
- const NativeExpertLayout L = native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
+ NativeExpertLayout L = native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
+ if (f.had_d) { L.had_block = f.had_block; L.d_signs = had_sh_; } // APR: h is rotated in there
native_expert_grouped(L, gp, gs, gn, p_dst, p_tok, cap, cap,
nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, dst_buf, cs, gy);
} else {
diff --git a/sycl/src/kernels/cuda/iq_kernels.dp.cpp b/sycl/src/kernels/cuda/iq_kernels.dp.cpp
index 1f8b3c718..313ed1186 100644
--- a/sycl/src/kernels/cuda/iq_kernels.dp.cpp
+++ b/sycl/src/kernels/cuda/iq_kernels.dp.cpp
@@ -18,6 +18,8 @@
#define GGML_COMMON_DECL_SYCL
#define GGML_COMMON_IMPL_SYCL
#include "ggml-common.h"
+#include "ggml-common-tq.h" // APR trellis types (agention llama.cpp fork)
+#include "strata/kernels/tq_lut_i8_sycl.hpp" // int8 copy of the trellis codebook (one global scale) for the dp4a dot
#include <sycl/ext/oneapi/matrix/matrix.hpp>
#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
@@ -27,6 +29,10 @@
#include <utility>
#include <cstdlib>
#include <type_traits>
+#include <map>
+#include <mutex>
+#include <tuple>
+#include <cmath>
namespace strata::kernels {
namespace {
@@ -728,6 +734,85 @@ __dpct_inline__ float vec_dot_q8_0_q8_1(const void *__restrict__ vbq,
return d8_0 * d8_1 * ((float) sumi);
}
+// ---------------------------------------------------------------- APR trellis types (ggml 144 TQ2_T, 145 TQK6, 146 TQK7)
+// Transcribed from src/kernels/cuda/iq_kernels.cu (which has them from the agention llama.cpp fork's ggml-cuda/tq.cuh,
+// 6d3f2b1b8; format in ggml-common-tq.h). A 128-value block is 8 lanes of 4 trellis steps (16 values); the dot of lane l
+// reads q8_1 block l/2, ints 4(l&1).. Same integer and float operations in the same order as the CUDA kernels: the
+// decode reads the int8 codebook tq_lut_i8 (one global scale) against q8_1 activations.
+template<int TY> struct TqInfo;
+template<> struct TqInfo<144> { using block_t = block_tq2_t; }; // byte-aligned, no K
+template<> struct TqInfo<145> { using block_t = block_tqk6; static constexpr int K = 6; };
+template<> struct TqInfo<146> { using block_t = block_tqk7; static constexpr int K = 7; };
+__dpct_inline__ float tq_h2f(uint32_t bits) { return (float) sycl::bit_cast<sycl::half>((uint16_t) bits); }
+// the four unscaled weights of the step with state s: (a.x, a.y, b.x, b.y)
+__dpct_inline__ void tq_step(uint32_t s, float (&w)[4]) {
+ const uint32_t* lut = (const uint32_t*) tq_lut_f16;
+ const uint32_t x = (s & TQ_STATE_MASK) * 0x9e3779b1u;
+ const uint32_t pa = lut[x >> 21], pb = lut[(x >> 10) & 2047u];
+ w[0] = tq_h2f(pa & 0xFFFFu);
+ w[1] = tq_h2f(pa >> 16);
+ w[2] = tq_h2f(pb & 0xFFFFu);
+ w[3] = tq_h2f(pb >> 16);
+}
+// the states of steps 4l..4l+3 of one block (qs is 2-byte aligned: every trellis block has an even size)
+template<int TY>
+__dpct_inline__ void tq_states4(const uint8_t* qs, int l, uint32_t s[4]) {
+ const uint16_t* qs16 = (const uint16_t*) qs;
+ if constexpr (TY == 144) { // s_t = qs[(t + 31) % 32] << 8 | qs[t]
+ const uint32_t w01 = qs16[2 * l + 0], w23 = qs16[2 * l + 1];
+ const uint32_t prev = qs[(4 * l + 31) & 31];
+ const uint32_t b0 = w01 & 0xFF, b1 = w01 >> 8, b2 = w23 & 0xFF, b3 = w23 >> 8;
+ s[0] = (prev << 8) | b0;
+ s[1] = (b0 << 8) | b1;
+ s[2] = (b1 << 8) | b2;
+ s[3] = (b2 << 8) | b3;
+ } else {
+ // step t's state: the 16 stream bits from bit (31 - t)*K, circular over 2K u16 words; steps 4l..4l+3 all
+ // lie in the four words from (28 - 4l)*K >> 4
+ constexpr int K = TqInfo<TY>::K, nwords = 2 * K;
+ const int off3 = (28 - 4 * l) * K, wb = off3 >> 4, r = off3 & 15;
+ const int wi1 = wb + 1 >= nwords ? wb + 1 - nwords : wb + 1;
+ const int wi2 = wb + 2 >= nwords ? wb + 2 - nwords : wb + 2;
+ const int wi3 = wb + 3 >= nwords ? wb + 3 - nwords : wb + 3;
+ const uint64_t v = (uint64_t) qs16[wb] | ((uint64_t) qs16[wi1] << 16) | ((uint64_t) qs16[wi2] << 32) |
+ ((uint64_t) qs16[wi3] << 48);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) s[i] = (uint32_t) (v >> (r + (3 - i) * K)) & 0xFFFFu;
+ }
+}
+// The lane's 16 codebook values as 4 ints of packed int8 (tq_load: the weight side only), the activation side in
+// tq_apply; vec_dot_tq_q8_1 is apply(load()). Numerics as in the CUDA file: the int8 codebook is within 0.85% RMS of
+// the fp16 one; the prompt path decodes through the fp16 codebook (dq_tq below).
+struct TqLane { int q[4]; float d; };
+template<int TY>
+__dpct_inline__ TqLane tq_load(const void* __restrict__ vbq, int kbx, int iqs) {
+ const auto* b = (const typename TqInfo<TY>::block_t*) vbq + kbx;
+ uint32_t s[4];
+ tq_states4<TY>(b->qs, iqs / 4, s);
+ TqLane r;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const uint32_t x = (s[i] & TQ_STATE_MASK) * 0x9e3779b1u;
+ r.q[i] = (int) ((uint32_t) tq_lut_i8[x >> 21] | ((uint32_t) tq_lut_i8[(x >> 10) & 2047u] << 16));
+ }
+ r.d = tq_h2f(b->d);
+ return r;
+}
+__dpct_inline__ float tq_apply(const TqLane& r, const block_q8_1* __restrict__ bq8_1, int iqs) {
+ const int l = iqs / 4;
+ const block_q8_1* b8 = bq8_1 + l / 2;
+ const int* q8 = (const int*) b8->qs + 4 * (l & 1);
+ int sumi = 0;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) sumi = ggml_cuda_dp4a(r.q[i], q8[i], sumi);
+ return r.d * ((float) b8->ds[0] * TQ_LUT_I8_SCALE) * (float) sumi;
+}
+template<int TY>
+__dpct_inline__ float vec_dot_tq_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
+ const int& kbx, const int& iqs) {
+ return tq_apply(tq_load<TY>(vbq, kbx, iqs), bq8_1, iqs);
+}
+
// ---------------------------------------------------------------- the formats
// qk = values per block, ipb = dot calls per block (qi / vdr), step = the iqs stride between calls.
template<int TY>
@@ -766,16 +851,23 @@ template<> struct Fmt<6> { static constexpr int qk = 32, ipb = QI5_0 / VDR_Q5_0,
static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_0_q8_1(v, y, kbx, iqs); } };
template<> struct Fmt<8> { static constexpr int qk = 32, ipb = QI8_0 / VDR_Q8_0, step = VDR_Q8_0;
static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q8_0_q8_1(v, y, kbx, iqs); } };
+// APR trellis types: 8 lanes of 4 steps per 128-value block, iqs = 4 * lane
+template<> struct Fmt<144> { static constexpr int qk = 128, ipb = 8, step = 4;
+ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<144>(v, y, kbx, iqs); } };
+template<> struct Fmt<145> { static constexpr int qk = 128, ipb = 8, step = 4;
+ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<145>(v, y, kbx, iqs); } };
+template<> struct Fmt<146> { static constexpr int qk = 128, ipb = 8, step = 4;
+ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<146>(v, y, kbx, iqs); } };
// The formats of each role, one list each so a type cannot be in one switch and missing from another. Every
// entry is a kernel template for each CUDA architecture of the build, hence two lists rather than one.
#ifdef STRATA_Q6K_EXPERTS // opt-in build (-DSTRATA_Q6K_EXPERTS=ON): one more instance per kernel, loaded at start
-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(14) X(6) X(2) X(3) X(8)
+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(14) X(6) X(2) X(3) X(8) X(144) X(145)
#else
-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(2) X(3) X(8)
+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(2) X(3) X(8) X(144) X(145)
#endif
-#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8)
-#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(2) X(3) X(8)
+#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8) X(144) X(146)
+#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(2) X(3) X(8) X(144) X(145) X(146)
__dpct_inline__ float warp_sum(float v) {
#pragma unroll
@@ -1212,6 +1304,22 @@ template<> struct Split<42> { // Q2_0
}
};
+// APR trellis types: the lane's 16 decoded codebook values (tq_load / tq_apply above)
+template<> inline constexpr bool kSplit<144> = true;
+template<> inline constexpr bool kSplit<145> = true;
+template<> inline constexpr bool kSplit<146> = true;
+template<int TY> struct SplitTq {
+ using W = TqLane;
+ template<bool STAGE_GRID = false> // (no grid table to stage: the codebook is read where it is)
+ static W load(const void* __restrict__ vbq, int kbx, int iqs, const uint32_t* __restrict__ = nullptr) {
+ return tq_load<TY>(vbq, kbx, iqs);
+ }
+ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) { return tq_apply(r, bq8_1, iqs); }
+};
+template<> struct Split<144> : SplitTq<144> {};
+template<> struct Split<145> : SplitTq<145> {};
+template<> struct Split<146> : SplitTq<146> {};
+
// One row against the n <= NC activations x + off[0..n) (n >= 1, warp-uniform; offsets in q8_1 blocks, 32-bit to
// spare registers), the whole warp. Per activation this is row_dot: the same calls k, lane-strided the same way,
// summed in the same order, then the same warp_sum. Only the weight side moves out of the per-activation loop.
@@ -2066,7 +2174,7 @@ __dpct_inline__ void native_gu_multi_kernel(
s_grid_buf, item_ct1.get_local_id(2), 256);
const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
if constexpr (SUB16) {
- if (xb == 80) {
+ if (xb == 80 && nb * Fmt<TG>::ipb == 80) { // row_dot_80_sub16 covers exactly 80 dots (not the 128-value TQ blocks)
const int subwarp = item_ct1.get_local_id(2) >> 4,
t = item_ct1.get_local_id(2) & 15;
const int row = item_ct1.get_group(2) * 16 + subwarp; // 0 .. 2*n_ff
@@ -2155,6 +2263,46 @@ __dpct_inline__ void native_gu_multi_kernel(
}
}
+// APR (prism.hadamard): y = H_block (signs * x) per block of each row, block = the work-group size (a power of two <= 1024),
+// one work-group per (row block, row). The butterflies are the CPU's (hadamard.cpp) in the same order, so the
+// rotated values are the same floats.
+template <typename T> __dpct_inline__ float ld_f(const T* p) { return (float) *p; }
+template <> __dpct_inline__ float ld_f<sycl::half>(const sycl::half* p) { return (float) *p; }
+template <typename T> __dpct_inline__ void st_f(T* p, float v) { *p = v; }
+template <> __dpct_inline__ void st_f<sycl::half>(sycl::half* p, float v) {
+ *p = sycl::vec<float, 1>(v).convert<sycl::half, sycl::rounding_mode::automatic>()[0];
+}
+template <typename T>
+__dpct_inline__ void hadamard_rows_kernel(const T* __restrict__ x, T* __restrict__ y, int64_t width,
+ const float* __restrict__ signs, float scale) {
+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
+ auto& hbuf = *sycl::ext::oneapi::group_local_memory_for_overwrite<float[1024]>(item_ct1.get_group());
+ const int n = (int) item_ct1.get_local_range(2), i = (int) item_ct1.get_local_id(2);
+ const int64_t col = (int64_t) item_ct1.get_group(2) * n + i, at = (int64_t) item_ct1.get_group(1) * width + col;
+ float v = ld_f<T>(x + at);
+ if (signs) v *= signs[col];
+ hbuf[i] = v;
+ for (int len = 1; len < n; len <<= 1) {
+ sycl::group_barrier(item_ct1.get_group());
+ if ((i & len) == 0) {
+ const float a = hbuf[i], c = hbuf[i + len];
+ hbuf[i] = a + c;
+ hbuf[i + len] = a - c;
+ }
+ }
+ sycl::group_barrier(item_ct1.get_group());
+ st_f<T>(y + at, hbuf[i] * scale);
+}
+template <typename T> class hadamard_rows_name;
+template <typename T>
+void launch_hadamard_rows(const T* x, T* y, int64_t rows, int64_t width, int block, const float* signs, dpct::queue_ptr s) {
+ const float scale = 1.0f / std::sqrt((float) block);
+ s->parallel_for<hadamard_rows_name<T>>(
+ sycl::nd_range<3>(sycl::range<3>(1, (size_t) rows, (size_t) (width / block) * (size_t) block),
+ sycl::range<3>(1, 1, (size_t) block)),
+ [=](sycl::nd_item<3>) [[sycl::reqd_sub_group_size(32)]] { hadamard_rows_kernel<T>(x, y, width, signs, scale); });
+}
+
__dpct_inline__ void swiglu_entries_kernel(const float *__restrict__ gate,
const float *__restrict__ up,
float *__restrict__ h, long long n) {
@@ -2921,6 +3069,29 @@ inline void dq_bf16(const void *vx, int64_t ibs, dst_t *yy, int tid) {
cvt<dst_t>(sycl::bit_cast<float>((uint32_t)x[j] << 16));
}
+// APR trellis types: one 256-value call = two 128-blocks; thread tid decodes block tid/16, lane (tid/2)%8, steps
+// 2(tid%2)..2(tid%2)+1 (8 values)
+template <int TY, typename dst_t>
+inline void dq_tq(const void *vx, int64_t ibs, dst_t *yy, int tid) {
+ const int blk = tid >> 4, l = (tid >> 1) & 7, half = tid & 1;
+ const auto* b = (const typename TqInfo<TY>::block_t*) vx + 2 * ibs + blk;
+ const float d = tq_h2f(b->d);
+ uint32_t s[4];
+ tq_states4<TY>(b->qs, l, s);
+ dst_t* y = yy + blk * 128 + 16 * l + 8 * half;
+ float v[8];
+#pragma unroll
+ for (int i = 0; i < 2; ++i) {
+ float w[4];
+ tq_step(s[2 * half + i], w);
+ v[4 * i + 0] = d * w[0];
+ v[4 * i + 1] = d * w[1];
+ v[4 * i + 2] = d * w[2];
+ v[4 * i + 3] = d * w[3];
+ }
+ store_run<dst_t, 8>(y, v);
+}
+
// Every type below must also be in is_iq() (BF16: embed_type_supported): the host entry points refuse the others,
// so the default is unreachable.
template <typename dst_t>
@@ -2951,6 +3122,9 @@ dq_dispatch(int ty, const void *vx, int64_t ibs, dst_t *y, int tid) {
case 3: dq_q4_1(vx, ibs, y, tid); break;
case 8: dq_q8_0(vx, ibs, y, tid); break;
case 30: dq_bf16(vx, ibs, y, tid); break;
+ case 144: dq_tq<144>(vx, ibs, y, tid); break;
+ case 145: dq_tq<145>(vx, ibs, y, tid); break;
+ case 146: dq_tq<146>(vx, ibs, y, tid); break;
default: break;
}
}
@@ -2978,7 +3152,7 @@ __dpct_inline__ void dequant_gu_kernel(
// the types dq_dispatch dequantizes
bool is_iq(int t) {
- return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||
+ return t == 144 || t == 145 || t == 146 || t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||
#ifdef STRATA_Q6K_EXPERTS
t == 12 || t == 13 || t == 14 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8;
#else
@@ -4454,6 +4628,58 @@ void iq_set_old_kernels(bool old) { g_old_kernels = old; }
void native_expert_set_kq_split(bool on) { g_kq_split = on; }
bool iq_old_kernels() { return g_old_kernels; }
+// APR (prism.hadamard): see iq_kernels.hpp. A block that is not a power of two in [2, 1024] dividing the row width exits with a
+// message, like check() does for a failed launch.
+namespace {
+void hadamard_bad_block(const char* what, int block, int64_t width) {
+ std::fprintf(stderr, "%s: Hadamard block %d is not a power of two in [2, 1024] dividing the row width %lld "
+ "(hadamard.txt's block and the expert widths disagree)\n", what, block, (long long) width);
+ std::exit(1);
+}
+constexpr int64_t kHadamardMaxRows = 65535; // the CUDA grid limit; a prompt chunk's rows go up in the same slices
+bool hadamard_block_ok(int block, int64_t width) {
+ return block >= 2 && block <= 1024 && (block & (block - 1)) == 0 && width % block == 0;
+}
+} // namespace
+
+void hadamard_rows(const float* x, float* y, int64_t rows, int64_t width, int block, const float* signs, void* stream) {
+ if (rows <= 0) return;
+ if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows", block, width);
+ dpct::queue_ptr s = strata::q_of(stream);
+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) {
+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);
+ launch_hadamard_rows<float>(x + r0 * width, y + r0 * width, n, width, block, signs, s);
+ }
+ check("hadamard_rows");
+}
+
+void hadamard_rows_f16(uint16_t* x, int64_t rows, int64_t width, int block, const float* signs, void* stream) {
+ if (rows <= 0) return;
+ if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows_f16", block, width);
+ dpct::queue_ptr s = strata::q_of(stream);
+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) {
+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);
+ launch_hadamard_rows<sycl::half>((const sycl::half*) x + r0 * width, (sycl::half*) x + r0 * width, n, width, block,
+ signs, s);
+ }
+ check("hadamard_rows_f16");
+}
+
+const float* hadamard_device_signs(const float* host, int width) {
+ if (host == nullptr || width <= 0) return nullptr;
+ static std::mutex mu;
+ static std::map<std::tuple<int, const float*, int>, float*> table; // (device, host vector, width)
+ const int dev = dpct::get_current_device_id();
+ std::lock_guard<std::mutex> lock(mu);
+ float*& d = table[std::make_tuple(dev, host, width)];
+ if (d == nullptr) {
+ sycl::queue& q = dpct::get_in_order_queue();
+ d = (float*) strata::malloc_device_guarded((size_t) width * sizeof(float), q, "Hadamard signs");
+ q.memcpy(d, host, (size_t) width * sizeof(float)).wait();
+ }
+ return d;
+}
+
bool iq_supported(int t) noexcept { return is_iq(t); }
bool embed_type_supported(int t) noexcept { return is_iq(t) || t == 30; }
@@ -4477,6 +4703,9 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {
case 2: return (size_t) (n / 32) * sizeof(block_q4_0);
case 3: return (size_t) (n / 32) * sizeof(block_q4_1);
case 8: return (size_t) (n / 32) * sizeof(block_q8_0);
+ case 144: return (size_t) (n / 128) * sizeof(block_tq2_t); // APR trellis types
+ case 145: return (size_t) (n / 128) * sizeof(block_tqk6);
+ case 146: return (size_t) (n / 128) * sizeof(block_tqk7);
case 30: return (size_t) n * 2; // BF16: the token embedding only (iq_embed_rows, iq_dequant_f32)
default: return 0;
}
@@ -5462,7 +5691,8 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
#else
const bool sw_v1 = v1;
#endif
- if (sw_v1) {
+ // APR: Hadamard-folded down weights take the rotated h, so h exists as floats first (the separate passes)
+ if (sw_v1 || L.had_block > 0) {
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
@@ -5477,6 +5707,8 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
swiglu_entries_kernel(gate, up, h, nh);
});
}
+ if (L.had_block > 0) // APR: the down weights are Hadamard-folded, so is their input (in place, every entry row)
+ hadamard_rows(h, h, cap_entries, L.n_ff, L.had_block, L.d_signs, stream);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
diff --git a/sycl/src/kernels/tq_sycl_parity.cpp b/sycl/src/kernels/tq_sycl_parity.cpp
new file mode 100644
index 000000000..c6a161bc5
--- /dev/null
+++ b/sycl/src/kernels/tq_sycl_parity.cpp
@@ -0,0 +1,366 @@
+// sycl/src/kernels/tq_sycl_parity.cpp - the Gyro trellis experts (TQ2_T 144, TQK6 145, TQK7 146, Hadamard-folded) on the
+// SYCL kernels, against real tensors of a Gyro GGUF.
+//
+// tq_sycl_parity <pack-dir> <model.gguf> [layer ...]
+// tq_sycl_parity --synthetic [--hadamard] GU/DOWN ... (random trellis codes and scales, e.g. 145/146 144/144)
+//
+// For one expert of each layer (the gate/up/down rows read from the GGUF, the Hadamard signs and block from the pack):
+// (a) float reference: fp16-codebook weights, x and h rotated in double, SwiGLU in double.
+// (b) the CPU engine path (native_quant_act, native_gu_rows, native_quant_h, native_down_rows).
+// (c) the GPU path (hadamard_rows, quantize_q8_1_rows, native_expert_grouped) with 3 entries in one group.
+// Checks: (c) against (a) and (b) within the bound every expert type gets (relative L1 < 3e-2); the GPU Hadamard rotation
+// against the CPU's; the gate/up rows and the down rows of (c) BIT FOR BIT against a host emulation of the kernels'
+// integer and float operations (the int8 codebook, the q8_1 blocks and the h the device produced are read back, the
+// lane-strided sums and the xor tree are replayed); the prompt-path dequantizers (iq_dequant_f32 / _f16) against the
+// fp16 codebook, bit for bit. Exit 0 only if all hold.
+#include "strata/artifact/gguf_reader.hpp"
+#include "strata/kernels/cpu/native_expert.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
+#include "strata/kernels/cpu/tq_cpu.hpp"
+#include "strata/kernels/iq_kernels.hpp"
+#include "ggml-common-tq.h"
+#include "../../../src/kernels/cpu/tq_states.hpp"
+#include "ggml.h"
+
+#include <sycl/sycl.hpp>
+#include <dpct/dpct.hpp>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+#define __device__
+#include "../../../src/kernels/cuda/tq_lut_i8.cuh"
+#undef __device__
+
+namespace cpu = strata::kernels::cpu;
+namespace {
+constexpr int NT = 3;
+constexpr int64_t H = 2560, FF = 640;
+constexpr int LN = 8; // STRATA_EXPERT_LANES
+
+double rel(const std::vector<float>& a, const std::vector<float>& b) {
+ double n = 0, d = 0;
+ for (size_t i = 0; i < a.size(); ++i) { n += std::fabs((double) a[i] - b[i]); d += std::fabs((double) b[i]); }
+ return n / (d + 1e-30);
+}
+float h2f(uint16_t b) { return cpu::tq_detail::tq_half_to_float(b); }
+int kbits(int t) { return t == 144 ? 8 : t == 145 ? 6 : 7; }
+size_t bsize(int t) { return 2 + 4 * (size_t) kbits(t); }
+
+// the 32 trellis states of block b (qs after the fp16 scale)
+void states(int t, const uint8_t* blk, uint32_t* st) {
+ if (t == 144) cpu::tq_detail::tq_states<8>(blk + 2, st);
+ else if (t == 145) cpu::tq_detail::tq_states<6>(blk + 2, st);
+ else cpu::tq_detail::tq_states<7>(blk + 2, st);
+}
+// float dequantization of one row of n values with the fp16 codebook
+void dq_row(int t, const uint8_t* row, int64_t n, float* y) {
+ for (int64_t b = 0; b < n / 128; ++b) {
+ const uint8_t* blk = row + b * bsize(t);
+ const float d = h2f((uint16_t) (blk[0] | (blk[1] << 8)));
+ uint32_t st[32];
+ states(t, blk, st);
+ for (int s = 0; s < 32; ++s) {
+ const uint32_t x = (st[s] & TQ_STATE_MASK) * 0x9e3779b1u;
+ const uint32_t pa = x >> 21, pb = (x >> 10) & 2047u;
+ y[b * 128 + 4 * s + 0] = d * h2f(tq_lut_f16[2 * pa]);
+ y[b * 128 + 4 * s + 1] = d * h2f(tq_lut_f16[2 * pa + 1]);
+ y[b * 128 + 4 * s + 2] = d * h2f(tq_lut_f16[2 * pb]);
+ y[b * 128 + 4 * s + 3] = d * h2f(tq_lut_f16[2 * pb + 1]);
+ }
+ }
+}
+// host replay of the kernels' dot of one row against one q8_1 row (36-byte blocks): lane `sub` of LN accumulates calls
+// sub, sub+LN, ... in order, then the xor tree (o = LN/2 .. 1)
+float emu_dot(int t, const uint8_t* row, const uint8_t* q8, int64_t n) {
+ const int nb = (int) (n / 128);
+ float s[LN] = {};
+ for (int sub = 0; sub < LN; ++sub)
+ for (int k = sub; k < nb * 8; k += LN) {
+ const int kbx = k / 8, lane = k % 8;
+ const uint8_t* blk = row + (size_t) kbx * bsize(t);
+ uint32_t st[32];
+ states(t, blk, st);
+ int sumi = 0;
+ const uint8_t* b8 = q8 + (size_t) (kbx * 4 + lane / 2) * 36;
+ for (int i = 0; i < 4; ++i) {
+ const uint32_t x = (st[4 * lane + i] & TQ_STATE_MASK) * 0x9e3779b1u;
+ const uint16_t p0 = tq_lut_i8[x >> 21], p1 = tq_lut_i8[(x >> 10) & 2047u];
+ const int8_t w[4] = {(int8_t) (p0 & 0xff), (int8_t) (p0 >> 8), (int8_t) (p1 & 0xff), (int8_t) (p1 >> 8)};
+ const int8_t* a = (const int8_t*) (b8 + 4 + 16 * (lane & 1) + 4 * i);
+ for (int j = 0; j < 4; ++j) sumi += (int) w[j] * (int) a[j];
+ }
+ const uint16_t dsb = (uint16_t) (b8[0] | (b8[1] << 8));
+ const float d = h2f((uint16_t) (blk[0] | (blk[1] << 8)));
+ volatile float tmp = h2f(dsb) * TQ_LUT_I8_SCALE;
+ volatile float v = d * tmp;
+ v = v * (float) sumi;
+ volatile float acc = s[sub];
+ acc = acc + v;
+ s[sub] = acc;
+ }
+ for (int o = LN / 2; o > 0; o >>= 1) {
+ float nx[LN];
+ for (int l = 0; l < LN; ++l) nx[l] = s[l] + s[l ^ o];
+ std::memcpy(s, nx, sizeof s);
+ }
+ return s[0];
+}
+
+template <class T> T* dmalloc(size_t n) { return (T*) sycl::malloc_device((n ? n : 1) * sizeof(T), dpct::get_in_order_queue()); }
+template <class T> void to_dev(T* d, const void* h, size_t n) { dpct::get_in_order_queue().memcpy(d, h, n).wait(); }
+template <class T> void from_dev(void* h, const T* d, size_t n) { dpct::get_in_order_queue().memcpy(h, d, n).wait(); }
+
+int check_layer(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int seed, const std::string& label) {
+ int failures = 0;
+ sycl::queue* s = &dpct::get_in_order_queue();
+ // (a) float reference (rows dequantized with the fp16 codebook)
+ std::vector<float> G((size_t) FF * H), U((size_t) FF * H), D((size_t) H * FF);
+ for (int64_t r = 0; r < FF; ++r) {
+ dq_row(f.gu_type, blob.data() + r * f.gu_row, H, G.data() + r * H);
+ dq_row(f.gu_type, blob.data() + f.up_off + r * f.gu_row, H, U.data() + r * H);
+ }
+ for (int64_t r = 0; r < H; ++r) dq_row(f.d_type, blob.data() + f.down_off + r * f.d_row, FF, D.data() + r * FF);
+ std::mt19937 rng(11 + seed);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::vector<float> x((size_t) NT * H);
+ for (auto& v : x) v = nd(rng);
+ std::vector<float> xr(x);
+ if (f.had_gu) for (int k = 0; k < NT; ++k) cpu::hadamard_rotate(xr.data() + k * H, (int) H, f.had_block, f.gu_signs);
+ std::vector<float> ref((size_t) NT * H), got_c((size_t) NT * H), got_g((size_t) NT * H);
+ for (int k = 0; k < NT; ++k) {
+ std::vector<float> h(FF);
+ for (int64_t r = 0; r < FF; ++r) {
+ double g = 0, u = 0;
+ for (int64_t i = 0; i < H; ++i) { g += (double) G[r * H + i] * xr[k * H + i]; u += (double) U[r * H + i] * xr[k * H + i]; }
+ h[r] = (float) ((g / (1.0 + std::exp(-g))) * u);
+ }
+ if (f.had_d) cpu::hadamard_rotate(h.data(), (int) FF, f.had_block, f.d_signs);
+ for (int64_t r = 0; r < H; ++r) {
+ double o = 0;
+ for (int64_t i = 0; i < FF; ++i) o += (double) D[r * FF + i] * h[i];
+ ref[k * H + r] = (float) o;
+ }
+ }
+ // (b) the CPU engine path
+ {
+ std::vector<std::vector<uint8_t>> act(NT, std::vector<uint8_t>(cpu::kNativeActBytes));
+ std::vector<std::vector<uint8_t>> hq(NT, std::vector<uint8_t>(cpu::kNativeHBytes));
+ std::vector<std::vector<float>> ff(NT, std::vector<float>(FF));
+ const void* a[NT];
+ float* ffp[NT];
+ const void* hp[NT];
+ float* op[NT];
+ for (int k = 0; k < NT; ++k) {
+ cpu::native_quant_act(f, x.data() + k * H, act[k].data());
+ a[k] = act[k].data();
+ ffp[k] = ff[k].data();
+ }
+ cpu::native_gu_rows(f, blob.data(), a, NT, ffp, 0, (int) FF);
+ for (int k = 0; k < NT; ++k) {
+ cpu::native_quant_h(f, ff[k].data(), hq[k].data());
+ hp[k] = hq[k].data();
+ op[k] = got_c.data() + k * H;
+ }
+ cpu::native_down_rows(f, blob.data(), hp, NT, op, 0, (int) H);
+ }
+ // (c) the GPU: one group holding the NT entries
+ auto L = strata::kernels::native_expert_layout(f.gu_type, f.d_type, H, FF);
+ if (f.had_d) { L.had_block = f.had_block; L.d_signs = strata::kernels::hadamard_device_signs(f.d_signs, (int) FF); }
+ uint8_t* dblob = dmalloc<uint8_t>(blob.size());
+ float* dx = dmalloc<float>(x.size());
+ uint8_t* dxq = dmalloc<uint8_t>((size_t) NT * H / 32 * 36);
+ const size_t scr_bytes = strata::kernels::native_expert_scratch_bytes(NT, FF);
+ uint8_t* dscr = dmalloc<uint8_t>(scr_bytes);
+ float* dout = dmalloc<float>((size_t) NT * H);
+ unsigned long long* dptr = dmalloc<unsigned long long>(1);
+ int32_t *dstart = dmalloc<int32_t>(2), *dn = dmalloc<int32_t>(1), *ddst = dmalloc<int32_t>(NT), *dtok = dmalloc<int32_t>(NT);
+ to_dev(dblob, blob.data(), blob.size());
+ to_dev(dx, x.data(), x.size() * 4);
+ const unsigned long long p = (unsigned long long) dblob;
+ const int32_t st2[2] = {0, NT}, one = 1, idx[NT] = {0, 1, 2};
+ to_dev(dptr, &p, 8);
+ to_dev(dstart, st2, 8);
+ to_dev(dn, &one, 4);
+ to_dev(ddst, idx, NT * 4);
+ to_dev(dtok, idx, NT * 4);
+ if (f.had_gu) {
+ strata::kernels::hadamard_rows(dx, dx, NT, H, f.had_block, strata::kernels::hadamard_device_signs(f.gu_signs, (int) H), s);
+ std::vector<float> gx(x.size());
+ s->wait();
+ from_dev(gx.data(), dx, gx.size() * 4);
+ const bool same = std::memcmp(gx.data(), xr.data(), gx.size() * 4) == 0;
+ std::printf(" gpu hadamard_rows vs cpu hadamard_rotate: %s (rel %.2e)\n", same ? "bitwise equal" : "differ", rel(gx, xr));
+ if (!(rel(gx, xr) < 1e-6)) ++failures;
+ std::vector<uint16_t> xh(x.size()), xh_out(x.size());
+ for (size_t i = 0; i < x.size(); ++i) xh[i] = ggml_fp32_to_fp16(x[i]);
+ uint16_t* dxh = dmalloc<uint16_t>(xh.size());
+ to_dev(dxh, xh.data(), xh.size() * 2);
+ strata::kernels::hadamard_rows_f16(dxh, NT, H, f.had_block, strata::kernels::hadamard_device_signs(f.gu_signs, (int) H), s);
+ s->wait();
+ from_dev(xh_out.data(), dxh, xh.size() * 2);
+ sycl::free(dxh, *s);
+ std::vector<float> xf(x.size());
+ for (size_t i = 0; i < xf.size(); ++i) xf[i] = ggml_fp16_to_fp32(xh_out[i]);
+ std::printf(" gpu hadamard_rows_f16 vs cpu float rotation: rel %.2e\n", rel(xf, xr));
+ if (!(rel(xf, xr) < 2e-3)) ++failures;
+ }
+ strata::kernels::quantize_q8_1_rows(dx, NT, H, dxq, s);
+ strata::kernels::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, 1, NT, dxq, dscr, dout, s);
+ s->wait();
+ from_dev(got_g.data(), dout, got_g.size() * 4);
+ { // bit-for-bit replay of the lane kernels
+ std::vector<uint8_t> hxq((size_t) NT * H / 32 * 36);
+ from_dev(hxq.data(), dxq, hxq.size());
+ const size_t fb = (size_t) NT * FF * 4, fa = (fb + 255) & ~(size_t) 255;
+ std::vector<float> gate((size_t) NT * FF), up((size_t) NT * FF), hh((size_t) NT * FF);
+ from_dev(gate.data(), (const float*) dscr, gate.size() * 4);
+ from_dev(up.data(), (const float*) (dscr + fa), up.size() * 4);
+ from_dev(hh.data(), (const float*) (dscr + 2 * fa), hh.size() * 4);
+ std::vector<uint8_t> hhq((size_t) NT * FF / 32 * 36);
+ from_dev(hhq.data(), dscr + 3 * fa, hhq.size());
+ size_t gu_diff = 0, dn_diff = 0;
+ for (int k = 0; k < NT; ++k) {
+ for (int64_t r = 0; r < FF; ++r) {
+ const float eg = emu_dot(f.gu_type, blob.data() + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);
+ const float eu = emu_dot(f.gu_type, blob.data() + f.up_off + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);
+ gu_diff += std::memcmp(&eg, &gate[k * FF + r], 4) != 0;
+ gu_diff += std::memcmp(&eu, &up[k * FF + r], 4) != 0;
+ }
+ for (int64_t r = 0; r < H; ++r) {
+ const float ed = emu_dot(f.d_type, blob.data() + f.down_off + r * f.d_row, hhq.data() + (size_t) k * FF / 32 * 36, FF);
+ dn_diff += std::memcmp(&ed, &got_g[k * H + r], 4) != 0;
+ }
+ }
+ std::printf(" kernel replay on the host (int8 codebook, q8_1, lane sums): gate/up %zu of %lld, down %zu of %lld values differ in any bit\n",
+ gu_diff, (long long) (2 * NT * FF), dn_diff, (long long) (NT * H));
+ if (gu_diff || dn_diff) ++failures;
+ (void) hh;
+ }
+ { // (d) the GPU dequantizers (prompt path) against the fp16-codebook floats
+ float* dq = dmalloc<float>((size_t) FF * H);
+ uint16_t* dh = dmalloc<uint16_t>((size_t) FF * H);
+ std::vector<float> got_dq((size_t) FF * H);
+ std::vector<uint16_t> got_h((size_t) FF * H);
+ size_t differ = 0, h_differ = 0;
+ const struct { const uint8_t* src; int type; const std::vector<float>* want; } mats[3] = {
+ {dblob, f.gu_type, &G}, {dblob + f.up_off, f.gu_type, &U}, {dblob + f.down_off, f.d_type, &D}};
+ for (const auto& m : mats) {
+ strata::kernels::iq_dequant_f32(m.type, m.src, FF * H, dq, s);
+ strata::kernels::iq_dequant_f16(m.type, m.src, FF * H, dh, s);
+ s->wait();
+ from_dev(got_dq.data(), dq, got_dq.size() * 4);
+ from_dev(got_h.data(), dh, got_h.size() * 2);
+ for (size_t i = 0; i < got_dq.size(); ++i) {
+ differ += std::memcmp(&got_dq[i], &(*m.want)[i], 4) != 0;
+ h_differ += got_h[i] != ggml_fp32_to_fp16(got_dq[i]);
+ }
+ }
+ sycl::free(dq, *s); sycl::free(dh, *s);
+ std::printf(" GPU dequant vs fp16-codebook floats: %zu of %lld values differ in any bit; fp16 of it: %zu differ\n",
+ differ, (long long) (3 * FF * H), h_differ);
+ if (differ || h_differ) ++failures;
+ }
+ sycl::free(dblob, *s); sycl::free(dx, *s); sycl::free(dxq, *s); sycl::free(dscr, *s); sycl::free(dout, *s);
+ sycl::free(dptr, *s); sycl::free(dstart, *s); sycl::free(dn, *s); sycl::free(ddst, *s); sycl::free(dtok, *s);
+ const double ec = rel(got_c, ref), eg = rel(got_g, ref), ecg = rel(got_c, got_g);
+ const bool ok = ec < 3e-2 && eg < 3e-2 && std::isfinite(ec) && std::isfinite(eg);
+ std::printf("%-9s types %d/%d blob %8zu cpu rel %.2e gpu rel %.2e cpu-gpu %.2e %s\n", label.c_str(), f.gu_type, f.d_type,
+ f.bytes, ec, eg, ecg, ok ? "ok" : "FAIL");
+ return failures + (ok ? 0 : 1);
+}
+} // namespace
+
+// random trellis bytes (every bit pattern is a valid trellis), scales in a sane fp16 range
+std::vector<uint8_t> synthetic_blob(const cpu::NativeFmt& f, int seed) {
+ std::mt19937 rng(1234 + seed);
+ std::vector<uint8_t> blob(f.bytes);
+ for (auto& b : blob) b = (uint8_t) rng();
+ auto fix = [&](uint8_t* base, int type, size_t row_bytes, int64_t rows) {
+ for (int64_t r = 0; r < rows; ++r)
+ for (size_t b = 0; b < row_bytes / bsize(type); ++b) {
+ uint8_t* blk = base + r * row_bytes + b * bsize(type);
+ const float d = 0.002f + 0.03f * ((rng() & 0xffff) / 65535.0f);
+ const uint16_t h = ggml_fp32_to_fp16(d);
+ blk[0] = (uint8_t) (h & 0xff);
+ blk[1] = (uint8_t) (h >> 8);
+ }
+ };
+ fix(blob.data(), f.gu_type, f.gu_row, FF);
+ fix(blob.data() + f.up_off, f.gu_type, f.gu_row, FF);
+ fix(blob.data() + f.down_off, f.d_type, f.d_row, H);
+ return blob;
+}
+int type_of(const std::string& s) { return std::atoi(s.c_str()); }
+
+int main(int argc, char** argv) {
+ setvbuf(stdout, nullptr, _IONBF, 0);
+ if (argc >= 2 && std::string(argv[1]) == "--synthetic") {
+ int failures = 0, i = 2;
+ const bool had = argc > 2 && std::string(argv[2]) == "--hadamard";
+ if (had) ++i;
+ static std::vector<float> sx((size_t) H), sh((size_t) FF);
+ std::mt19937 srng(5);
+ for (auto& v : sx) v = (srng() & 1) ? 1.f : -1.f;
+ for (auto& v : sh) v = (srng() & 1) ? 1.f : -1.f;
+ for (; i < argc; ++i) {
+ const std::string arg = argv[i];
+ const size_t slash = arg.find('/');
+ const int gu = type_of(arg.substr(0, slash)), dn = type_of(arg.substr(slash + 1));
+ cpu::NativeFmt f;
+ std::string err;
+ if (slash == std::string::npos || !cpu::native_fmt(gu, dn, H, FF, f, err) ||
+ !strata::kernels::native_expert_supported(gu, dn, H, FF)) {
+ std::printf("%s: %s\n", arg.c_str(), err.empty() ? "not a pair the GPU expert kernels take" : err.c_str());
+ ++failures;
+ continue;
+ }
+ if (had) {
+ f.had_block = 128;
+ f.had_gu = f.had_d = true;
+ f.gu_signs = sx.data();
+ f.d_signs = sh.data();
+ }
+ failures += check_layer(f, synthetic_blob(f, i), i, had ? "hadamard" : "synthetic");
+ }
+ std::printf("tq_sycl_parity: %d failures\n", failures);
+ return failures ? 1 : 0;
+ }
+ if (argc < 3) { std::fprintf(stderr, "usage: tq_sycl_parity <pack-dir> <model.gguf> [layer ...]\n"); return 2; }
+ std::string err;
+ if (!cpu::expert_layout_load(argv[1], 48, 512, err)) { std::printf("expert_layout_load: %s\n", err.c_str()); return 1; }
+ const auto& lay = cpu::expert_layout();
+ const strata::GgufModel model(strata::gguf_split_paths(argv[2]));
+ std::vector<int> layers;
+ for (int i = 3; i < argc; ++i) layers.push_back(std::atoi(argv[i]));
+ if (layers.empty()) layers = {0, 1, 2, 3, 20, 47};
+ int failures = 0;
+ for (int l : layers) {
+ const strata::TensorInfo* t[3] = {};
+ const uint8_t* data[3] = {};
+ const char* roles[3] = {"gate", "up", "down"};
+ for (int r = 0; r < 3; ++r) {
+ size_t at = 0;
+ t[r] = model.find("blk." + std::to_string(l) + ".ffn_" + roles[r] + "_exps.weight", &at);
+ if (t[r]) data[r] = model.shard(at).tensor_data(*t[r]);
+ }
+ if (!t[0] || !t[1] || !t[2]) { std::printf("layer %d: no expert tensors\n", l); ++failures; continue; }
+ cpu::NativeFmt f = lay.fmt[(size_t) l];
+ for (int E : {7, 100}) {
+ std::vector<uint8_t> blob(f.bytes);
+ std::memcpy(blob.data(), data[0] + (size_t) E * f.up_off, f.up_off);
+ std::memcpy(blob.data() + f.up_off, data[1] + (size_t) E * f.up_off, f.up_off);
+ std::memcpy(blob.data() + f.down_off, data[2] + (size_t) E * (f.bytes - f.down_off), f.bytes - f.down_off);
+ failures += check_layer(f, blob, l * 1000 + E, "layer " + std::to_string(l) + " e" + std::to_string(E));
+ }
+ }
+ std::printf("tq_sycl_parity: %d failures\n", failures);
+ return failures ? 1 : 0;
+}
diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cpp
index 92cedf489..28222d91a 100644
--- a/sycl/src/prefill/prefill.cpp
+++ b/sycl/src/prefill/prefill.cpp
@@ -24,6 +24,7 @@
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/iq_dequant_expert.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
#include "strata/kernels/qsa.hpp"
#include "strata/kernels/kv_stream.hpp"
#include "strata/kernels/cvec.hpp"
@@ -715,6 +716,9 @@ struct Prefill::Impl {
uint16_t *xn16 = nullptr, *lo16 = nullptr;
float* mixed = nullptr;
uint16_t *mixed_bf = nullptr, *mixed_h = nullptr;
+ // APR (prism.hadamard): per layer, this device's copies of the gate/up (n_embd) and down (n_ff) sign vectors
+ // (nullptr: identity signs or not folded), resolved once in init so a prompt chunk reads a pointer
+ std::vector<const float*> had_sx, had_sh;
bool f16_io = false; // HIP, STRATA_HIP_PROMPT_F16=1: this stage's 16-bit GEMMs run FP16 in and out (set in init)
uint16_t *xn16_lo = nullptr, *lo16_lo = nullptr, *mixed_bf_lo = nullptr; // bf16x2(): the BF16 GEMMs' low parts
float* bo = nullptr;
@@ -1187,6 +1191,17 @@ bool Prefill::init(const core::WeightTable &wt, const core::ModelGeometry &g,
if (g.n_embd != N || g.hc != HC || g.hc_lr != LR || g.n_expert < 1 || ss.k != K) {
err = "prefill: geometry differs from the artifact's"; return false;
}
+ { // APR: the sign vectors on this device, once (hadamard_device_signs takes a mutex and allocates its copy)
+ const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();
+ m.had_sx.assign(lay.native ? lay.fmt.size() : 0, nullptr);
+ m.had_sh.assign(m.had_sx.size(), nullptr);
+ for (size_t l = 0; l < m.had_sx.size(); ++l) {
+ if (lay.fmt[l].had_gu)
+ m.had_sx[l] = strata::kernels::hadamard_device_signs(lay.fmt[l].gu_signs, (int) lay.fmt[l].n_embd);
+ if (lay.fmt[l].had_d)
+ m.had_sh[l] = strata::kernels::hadamard_device_signs(lay.fmt[l].d_signs, (int) lay.fmt[l].n_ff);
+ }
+ }
m.device = dpct::get_current_device_id();
if (static bool said_i8 = false; !said_i8 && i8_mode(m.cs) != 0) {
said_i8 = true;
@@ -3471,6 +3486,17 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
// (moe_fused_iq.hpp) where they cover its two formats, else MMQ (or the FP16 path: IQ1_M).
const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();
const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];
+ // APR (prism.hadamard): folded gate/up take H (s * x) on the gathered fp16 rows, folded down H (s * h) on
+ // the fp16 SwiGLU output, both rotated in fp16 storage with fp32 arithmetic (as the CUDA FP16 path does).
+ // The fused int8 products and the MMQ path do not rotate; they take only formats of unfolded layers.
+ const bool had_gu = lay.native && lay.fmt[(size_t) l].had_gu;
+ const bool had_d = lay.native && lay.fmt[(size_t) l].had_d;
+ const int had_block = lay.native ? lay.fmt[(size_t) l].had_block : 0;
+ const float* had_sx = had_gu ? m.had_sx[(size_t) l] : nullptr;
+ const float* had_sh = had_d ? m.had_sh[(size_t) l] : nullptr;
+ auto had_h = [&](int64_t row0, int64_t nrows) {
+ if (had_d) strata::kernels::hadamard_rows_f16(m.Hh + row0 * 640, nrows, 640, had_block, had_sh, m.cs);
+ };
const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42;
const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42;
// --peer-device: MMQ only, whether or not the peer took the prompt path (set_peer can decline), as
@@ -3715,6 +3741,8 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
}
} else {
gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs);
+ // APR: Xs is this layer's private gathered copy (rotated in fp16 storage, fp32 arithmetic)
+ if (had_gu) strata::kernels::hadamard_rows_f16(m.Xs, T * K, N, had_block, had_sx, m.cs);
}
// multi-GPU: the peer's share, enqueued before the primary's own experts so both cards work at once
peer_now = use_mmq && !order_peer.empty();
@@ -4195,11 +4223,11 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
const int gt = lay.fmt[(size_t) l].gu_type, dt = lay.fmt[(size_t) l].d_type;
return (gt == 18 || gt == 21 || gt == 22 || gt == 23) && (dt == 20 || dt == 42);
};
- const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt();
+ const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt() && !had_d;
// 151-b70: the fused int8 DPAS products (IQ3_S gate/up, IQ4_NL down, the 640 x 2560 expert), the same
// held group of staged experts as above
const int i8m = i8_mode(m.cs);
- const bool i8on = !xmx2_only && i8m != 0 && !use_mmq && lay.native &&
+ const bool i8on = !xmx2_only && i8m != 0 && !use_mmq && lay.native && !had_gu && !had_d &&
(lay.fmt[(size_t) l].gu_type == 21 || lay.fmt[(size_t) l].gu_type == 18 || lay.fmt[(size_t) l].gu_type == 22) &&
(lay.fmt[(size_t) l].d_type == 20 || lay.fmt[(size_t) l].d_type == 42) &&
lay.fmt[(size_t) l].n_ff == 640 && lay.fmt[(size_t) l].n_embd == 2560 &&
@@ -4293,6 +4321,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
pt.mark(kPfGemmGU, cs);
m.gemm.f16(m.Xs + o * N, m.dq_gu[qd], m.GU + o * 1280, xg_c[i], 1280, N);
swiglu_interleaved(m.GU + o * 1280, m.Hh + o * 640, xg_c[i], m.cs);
+ had_h(o, xg_c[i]);
pt.mark(kPfGemmD, cs);
m.gemm.f16(m.Hh + o * 640, m.dq_d[qd], m.Dm + o * N, xg_c[i], N, 640);
o += xg_c[i];
@@ -4322,6 +4351,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
strata::kernels::iq_xmx_grouped(f.gu_type, xg_g, xg_u, xg_c, xg_n, m.Xs + xg_r0 * N,
m.GU + xg_r0 * 1280, 1280, (int) N, m.cs);
swiglu_interleaved(m.GU + xg_r0 * 1280, m.Hh + xg_r0 * 640, nr, m.cs);
+ had_h(xg_r0, nr);
pt.mark(kPfGemmD, cs);
strata::kernels::iq_xmx_grouped(f.d_type, xg_d, nullptr, xg_c, xg_n, m.Hh + xg_r0 * 640,
m.Dm + xg_r0 * N, (int) N, 640, m.cs);
@@ -4336,6 +4366,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
pt.mark(kPfGemmGU, cs);
m.gemm.f16(m.Xs + o * N, m.dq_gu[q], m.GU + o * 1280, xg_c[i], 1280, N);
swiglu_interleaved(m.GU + o * 1280, m.Hh + o * 640, xg_c[i], m.cs);
+ had_h(o, xg_c[i]);
pt.mark(kPfGemmD, cs);
m.gemm.f16(m.Hh + o * 640, m.dq_d[q], m.Dm + o * N, xg_c[i], N, 640);
o += xg_c[i];
@@ -4473,6 +4504,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
}
}
swiglu_interleaved(m.GU + r0 * 1280, m.Hh + r0 * 640, nr, m.cs);
+ had_h(r0, nr);
pt.mark(kPfGemmD, cs);
strata::prefill::xmx::grouped_f16(m.Hh + r0 * 640, wd, gc, ng, m.Dm + r0 * N, (int) N, 640,
m.cs);
@@ -4482,6 +4514,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
pt.mark(kPfGemmGU, cs);
m.gemm.f16(m.Xs + o0 * N, m.dq_gu[q], m.GU + o0 * 1280, ne, 1280, N);
swiglu_interleaved(m.GU + o0 * 1280, m.Hh + o0 * 640, ne, m.cs);
+ had_h(o0, ne);
pt.mark(kPfGemmD, cs);
m.gemm.f16(m.Hh + o0 * 640, m.dq_d[q], m.Dm + o0 * N, ne, N, 640);
return true;