Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
SYCL: trellis experts (TQ2_T, TQK6, TQK7) for decode and the prompt path, Hadamard-folded inputs; tq_sycl_parity (host replay of the kernels bit for bit, dequantizers, Hadamard)
Niko122102791a6d18fb
diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txtindex b5eb3f6b5..4dc7ae000 100644--- a/sycl/CMakeLists.txt+++ b/sycl/CMakeLists.txt@@ -424,6 +424,11 @@ if(STRATA_SYCL_PARITY)add_executable(native_expert_parity ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/native_expert_parity.cpp)target_include_directories(native_expert_parity PRIVATE ${STRATA_ROOT}/third_party/ggml)target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu)+ # Gyro (APR) trellis experts on real GGUF tensors: tq_sycl_parity <pack> <model.gguf> [layer ...]+ add_executable(tq_sycl_parity ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/tq_sycl_parity.cpp)+ target_include_directories(tq_sycl_parity PRIVATE ${STRATA_ROOT}/third_party/ggml)+ target_compile_options(tq_sycl_parity PRIVATE -ffp-contract=off)+ target_link_libraries(tq_sycl_parity PRIVATE strata_engine strata_kernels strata_kernels_cpu)# MODEL-UPDATE: the CPU pool's expert rows per format against the RAM bandwidth (as the top-level build's)add_executable(kq_cpu_bench ${STRATA_ROOT}/src/kernels/cpu/kq_cpu_bench.cpp)target_link_libraries(kq_cpu_bench PRIVATE strata_kernels_cpu)diff --git a/sycl/include/strata/core/verify.hpp b/sycl/include/strata/core/verify.hppindex 894f7b7b7..b38249f4d 100644--- a/sycl/include/strata/core/verify.hpp+++ b/sycl/include/strata/core/verify.hpp@@ -460,6 +460,10 @@ private:static constexpr int64_t kPcieGroupRows = 4; // the PCIe call's groups side by side (of <= 16)uint8_t* hit_xq_ = nullptr;uint8_t* nat_xq_ = nullptr; // plan v0.3 P6: q8_1 activations for a native pack's grouped experts+ float* nat_xr_ = nullptr; // APR: the Hadamard-rotated expert input (the shared expert and the CPU keep xm); nullptr+ // unless some layer's gate/up is folded+ const float* had_sx_ = nullptr; // APR: device sign vectors for x (n_embd) and h (n_ff); nullptr = identity+ const float* had_sh_ = nullptr;float* hit_xs_ = nullptr;void* hit_scratch_ = nullptr;float *head_mixed_ = nullptr, *head_inj_ = nullptr, *head_logits_ = nullptr;diff --git a/sycl/include/strata/kernels/tq_lut_i8_sycl.hpp b/sycl/include/strata/kernels/tq_lut_i8_sycl.hppnew file mode 100644index 000000000..b63713e82--- /dev/null+++ b/sycl/include/strata/kernels/tq_lut_i8_sycl.hpp@@ -0,0 +1,10 @@+// The int8 copy of the trellis codebook (src/kernels/cuda/tq_lut_i8.cuh, unchanged) for the SYCL kernels: a const+// table at namespace scope, which a SYCL kernel reads like ggml's i-quant grids. The .cuh marks it __device__ for+// CUDA/HIP; here that qualifier is empty for the length of the include.+#pragma once+#ifdef __device__+#error "tq_lut_i8_sycl.hpp: __device__ is already defined"+#endif+#define __device__+#include "../../../../src/kernels/cuda/tq_lut_i8.cuh"+#undef __device__diff --git a/sycl/src/core/verify.cpp b/sycl/src/core/verify.cppindex 1d7501f11..a20208624 100644--- a/sycl/src/core/verify.cpp+++ b/sycl/src/core/verify.cpp@@ -15,6 +15,7 @@#include "strata/core/peer_experts.hpp"#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"#include "strata/kernels/bf16_gemv.hpp"#include "strata/kernels/native_router.hpp"#include "strata/kernels/native_moe.hpp"@@ -562,6 +563,13 @@ bool Verifier::init(const WeightTable &wt, const ModelGeometry &g,sink_.ctx = this;}+ // APR: the rotated copy of the expert input (nat_xr_, T x N floats) only where a layer's gate/up is+ // Hadamard-folded; its one reader is that layer's f.had_gu branch in the expert-input step below+ bool had_gu_any = false;+ {+ const auto& lay = strata::kernels::cpu::expert_layout();+ for (size_t l = 0; lay.native && l < lay.fmt.size(); ++l) had_gu_any = had_gu_any || lay.fmt[l].had_gu;+ }// ---- the device arena: the same sequence counted, then carvedauto carve = [&](Bump& b) {tok_ = b.take<int32_t>(T); step_ = b.take<int32_t>(T * strata::kernels::kStepCount);@@ -589,6 +597,7 @@ bool Verifier::init(const WeightTable &wt, const ModelGeometry &g,staging_ = b.take<uint8_t>((uint64_t) kStagingBlobs * strata::kernels::cpu::expert_layout().max_blob);hit_xq_ = b.take<uint8_t>(T * (N / 32) * 34); hit_xs_ = b.take<float>(T * (N / 32));nat_xq_ = b.take<uint8_t>(T * (N / 32) * 36);+ nat_xr_ = had_gu_any ? b.take<float>(T * N) : nullptr;hit_scratch_ = b.take<uint8_t>(std::max<uint64_t>(strata::kernels::moe_hit_grouped_scratch_bytes((int64_t) (T * K), g.n_embd, g.n_ff),strata::kernels::native_expert_scratch_bytes((int64_t) (T * K), g.n_ff)));@@ -652,6 +661,10 @@ bool Verifier::init(const WeightTable &wt, const ModelGeometry &g,return false;}}+ { // APR (prism.hadamard, hadamard.txt): the sign vectors on this device, before any graph capture+ had_sx_ = strata::kernels::hadamard_device_signs(strata::kernels::cpu::hadamard_signs((int) g.n_embd), (int) g.n_embd);+ had_sh_ = strata::kernels::hadamard_device_signs(strata::kernels::cpu::hadamard_signs((int) g.n_ff), (int) g.n_ff);+ }sink_.staging = (unsigned long long) staging_;sink_.staging_cap = kStagingBlobs;(void) TS;@@ -1395,8 +1408,17 @@ bool Verifier::record_window(int T, dpct::queue_ptr cs, std::string &err) {if (sh_fork) dpct::sync_barrier(ev_join_, sh_cs_);}if (strata::kernels::cpu::expert_layout().native) {- if (!qdedup)- if (!q8_ffn) quantize_q8_1_rows(xm, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);+ // APR: Hadamard-folded gate/up take H (s * x); xm itself stays as it is (the shared expert above and the+ // doorbell copy for the CPU pool, which rotates its own in native_quant_act). The q8_1 image of xm that+ // STRATA_QFUSE / STRATA_VERIFY_QDEDUP may already have written is of the unrotated input, so a folded layer+ // always rewrites it from the rotated rows (same stream, after the shared expert has read it).+ const auto& f = strata::kernels::cpu::expert_layout().fmt[(size_t) l];+ if (f.had_gu) { // nat_xr_ exists: init carved it because this layer is folded (had_gu_any)+ hadamard_rows(xm, nat_xr_ + (size_t) tb * N, n, N, f.had_block, had_sx_, cs);+ quantize_q8_1_rows(nat_xr_ + (size_t) tb * N, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);+ } else if (!qdedup && !q8_ffn) {+ quantize_q8_1_rows(xm, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);+ }} elsequantize_q8_0_scaled(xm, hit_xq_ + (size_t) tb * (N / 32) * 34, hit_xs_ + (size_t) tb * (N / 32), (int64_t) n * N, cs);stamp(l, 18, grp);@@ -1432,7 +1454,8 @@ bool Verifier::record_window(int T, dpct::queue_ptr cs, std::string &err) {if (lay.native) {// the layer's GGUF formats (i-quant gate/up, Q2_0 / IQ4_NL down)const auto& f = lay.fmt[(size_t) l];- const NativeExpertLayout L = native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);+ NativeExpertLayout L = native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);+ if (f.had_d) { L.had_block = f.had_block; L.d_signs = had_sh_; } // APR: h is rotated in therenative_expert_grouped(L, gp, gs, gn, p_dst, p_tok, cap, cap,nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, dst_buf, cs, gy);} else {diff --git a/sycl/src/kernels/cuda/iq_kernels.dp.cpp b/sycl/src/kernels/cuda/iq_kernels.dp.cppindex 1f8b3c718..313ed1186 100644--- a/sycl/src/kernels/cuda/iq_kernels.dp.cpp+++ b/sycl/src/kernels/cuda/iq_kernels.dp.cpp@@ -18,6 +18,8 @@#define GGML_COMMON_DECL_SYCL#define GGML_COMMON_IMPL_SYCL#include "ggml-common.h"+#include "ggml-common-tq.h" // APR trellis types (agention llama.cpp fork)+#include "strata/kernels/tq_lut_i8_sycl.hpp" // int8 copy of the trellis codebook (one global scale) for the dp4a dot#include <sycl/ext/oneapi/matrix/matrix.hpp>#include <sycl/ext/intel/experimental/grf_size_properties.hpp>@@ -27,6 +29,10 @@#include <utility>#include <cstdlib>#include <type_traits>+#include <map>+#include <mutex>+#include <tuple>+#include <cmath>namespace strata::kernels {namespace {@@ -728,6 +734,85 @@ __dpct_inline__ float vec_dot_q8_0_q8_1(const void *__restrict__ vbq,return d8_0 * d8_1 * ((float) sumi);}+// ---------------------------------------------------------------- APR trellis types (ggml 144 TQ2_T, 145 TQK6, 146 TQK7)+// Transcribed from src/kernels/cuda/iq_kernels.cu (which has them from the agention llama.cpp fork's ggml-cuda/tq.cuh,+// 6d3f2b1b8; format in ggml-common-tq.h). A 128-value block is 8 lanes of 4 trellis steps (16 values); the dot of lane l+// reads q8_1 block l/2, ints 4(l&1).. Same integer and float operations in the same order as the CUDA kernels: the+// decode reads the int8 codebook tq_lut_i8 (one global scale) against q8_1 activations.+template<int TY> struct TqInfo;+template<> struct TqInfo<144> { using block_t = block_tq2_t; }; // byte-aligned, no K+template<> struct TqInfo<145> { using block_t = block_tqk6; static constexpr int K = 6; };+template<> struct TqInfo<146> { using block_t = block_tqk7; static constexpr int K = 7; };+__dpct_inline__ float tq_h2f(uint32_t bits) { return (float) sycl::bit_cast<sycl::half>((uint16_t) bits); }+// the four unscaled weights of the step with state s: (a.x, a.y, b.x, b.y)+__dpct_inline__ void tq_step(uint32_t s, float (&w)[4]) {+ const uint32_t* lut = (const uint32_t*) tq_lut_f16;+ const uint32_t x = (s & TQ_STATE_MASK) * 0x9e3779b1u;+ const uint32_t pa = lut[x >> 21], pb = lut[(x >> 10) & 2047u];+ w[0] = tq_h2f(pa & 0xFFFFu);+ w[1] = tq_h2f(pa >> 16);+ w[2] = tq_h2f(pb & 0xFFFFu);+ w[3] = tq_h2f(pb >> 16);+}+// the states of steps 4l..4l+3 of one block (qs is 2-byte aligned: every trellis block has an even size)+template<int TY>+__dpct_inline__ void tq_states4(const uint8_t* qs, int l, uint32_t s[4]) {+ const uint16_t* qs16 = (const uint16_t*) qs;+ if constexpr (TY == 144) { // s_t = qs[(t + 31) % 32] << 8 | qs[t]+ const uint32_t w01 = qs16[2 * l + 0], w23 = qs16[2 * l + 1];+ const uint32_t prev = qs[(4 * l + 31) & 31];+ const uint32_t b0 = w01 & 0xFF, b1 = w01 >> 8, b2 = w23 & 0xFF, b3 = w23 >> 8;+ s[0] = (prev << 8) | b0;+ s[1] = (b0 << 8) | b1;+ s[2] = (b1 << 8) | b2;+ s[3] = (b2 << 8) | b3;+ } else {+ // step t's state: the 16 stream bits from bit (31 - t)*K, circular over 2K u16 words; steps 4l..4l+3 all+ // lie in the four words from (28 - 4l)*K >> 4+ constexpr int K = TqInfo<TY>::K, nwords = 2 * K;+ const int off3 = (28 - 4 * l) * K, wb = off3 >> 4, r = off3 & 15;+ const int wi1 = wb + 1 >= nwords ? wb + 1 - nwords : wb + 1;+ const int wi2 = wb + 2 >= nwords ? wb + 2 - nwords : wb + 2;+ const int wi3 = wb + 3 >= nwords ? wb + 3 - nwords : wb + 3;+ const uint64_t v = (uint64_t) qs16[wb] | ((uint64_t) qs16[wi1] << 16) | ((uint64_t) qs16[wi2] << 32) |+ ((uint64_t) qs16[wi3] << 48);+#pragma unroll+ for (int i = 0; i < 4; ++i) s[i] = (uint32_t) (v >> (r + (3 - i) * K)) & 0xFFFFu;+ }+}+// The lane's 16 codebook values as 4 ints of packed int8 (tq_load: the weight side only), the activation side in+// tq_apply; vec_dot_tq_q8_1 is apply(load()). Numerics as in the CUDA file: the int8 codebook is within 0.85% RMS of+// the fp16 one; the prompt path decodes through the fp16 codebook (dq_tq below).+struct TqLane { int q[4]; float d; };+template<int TY>+__dpct_inline__ TqLane tq_load(const void* __restrict__ vbq, int kbx, int iqs) {+ const auto* b = (const typename TqInfo<TY>::block_t*) vbq + kbx;+ uint32_t s[4];+ tq_states4<TY>(b->qs, iqs / 4, s);+ TqLane r;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const uint32_t x = (s[i] & TQ_STATE_MASK) * 0x9e3779b1u;+ r.q[i] = (int) ((uint32_t) tq_lut_i8[x >> 21] | ((uint32_t) tq_lut_i8[(x >> 10) & 2047u] << 16));+ }+ r.d = tq_h2f(b->d);+ return r;+}+__dpct_inline__ float tq_apply(const TqLane& r, const block_q8_1* __restrict__ bq8_1, int iqs) {+ const int l = iqs / 4;+ const block_q8_1* b8 = bq8_1 + l / 2;+ const int* q8 = (const int*) b8->qs + 4 * (l & 1);+ int sumi = 0;+#pragma unroll+ for (int i = 0; i < 4; ++i) sumi = ggml_cuda_dp4a(r.q[i], q8[i], sumi);+ return r.d * ((float) b8->ds[0] * TQ_LUT_I8_SCALE) * (float) sumi;+}+template<int TY>+__dpct_inline__ float vec_dot_tq_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,+ const int& kbx, const int& iqs) {+ return tq_apply(tq_load<TY>(vbq, kbx, iqs), bq8_1, iqs);+}+// ---------------------------------------------------------------- the formats// qk = values per block, ipb = dot calls per block (qi / vdr), step = the iqs stride between calls.template<int TY>@@ -766,16 +851,23 @@ template<> struct Fmt<6> { static constexpr int qk = 32, ipb = QI5_0 / VDR_Q5_0,static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_0_q8_1(v, y, kbx, iqs); } };template<> struct Fmt<8> { static constexpr int qk = 32, ipb = QI8_0 / VDR_Q8_0, step = VDR_Q8_0;static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q8_0_q8_1(v, y, kbx, iqs); } };+// APR trellis types: 8 lanes of 4 steps per 128-value block, iqs = 4 * lane+template<> struct Fmt<144> { static constexpr int qk = 128, ipb = 8, step = 4;+ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<144>(v, y, kbx, iqs); } };+template<> struct Fmt<145> { static constexpr int qk = 128, ipb = 8, step = 4;+ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<145>(v, y, kbx, iqs); } };+template<> struct Fmt<146> { static constexpr int qk = 128, ipb = 8, step = 4;+ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<146>(v, y, kbx, iqs); } };// The formats of each role, one list each so a type cannot be in one switch and missing from another. Every// entry is a kernel template for each CUDA architecture of the build, hence two lists rather than one.#ifdef STRATA_Q6K_EXPERTS // opt-in build (-DSTRATA_Q6K_EXPERTS=ON): one more instance per kernel, loaded at start-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(14) X(6) X(2) X(3) X(8)+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(14) X(6) X(2) X(3) X(8) X(144) X(145)#else-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(2) X(3) X(8)+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(2) X(3) X(8) X(144) X(145)#endif-#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8)-#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(2) X(3) X(8)+#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8) X(144) X(146)+#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(2) X(3) X(8) X(144) X(145) X(146)__dpct_inline__ float warp_sum(float v) {#pragma unroll@@ -1212,6 +1304,22 @@ template<> struct Split<42> { // Q2_0}};+// APR trellis types: the lane's 16 decoded codebook values (tq_load / tq_apply above)+template<> inline constexpr bool kSplit<144> = true;+template<> inline constexpr bool kSplit<145> = true;+template<> inline constexpr bool kSplit<146> = true;+template<int TY> struct SplitTq {+ using W = TqLane;+ template<bool STAGE_GRID = false> // (no grid table to stage: the codebook is read where it is)+ static W load(const void* __restrict__ vbq, int kbx, int iqs, const uint32_t* __restrict__ = nullptr) {+ return tq_load<TY>(vbq, kbx, iqs);+ }+ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) { return tq_apply(r, bq8_1, iqs); }+};+template<> struct Split<144> : SplitTq<144> {};+template<> struct Split<145> : SplitTq<145> {};+template<> struct Split<146> : SplitTq<146> {};+// One row against the n <= NC activations x + off[0..n) (n >= 1, warp-uniform; offsets in q8_1 blocks, 32-bit to// spare registers), the whole warp. Per activation this is row_dot: the same calls k, lane-strided the same way,// summed in the same order, then the same warp_sum. Only the weight side moves out of the per-activation loop.@@ -2066,7 +2174,7 @@ __dpct_inline__ void native_gu_multi_kernel(s_grid_buf, item_ct1.get_local_id(2), 256);const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);if constexpr (SUB16) {- if (xb == 80) {+ if (xb == 80 && nb * Fmt<TG>::ipb == 80) { // row_dot_80_sub16 covers exactly 80 dots (not the 128-value TQ blocks)const int subwarp = item_ct1.get_local_id(2) >> 4,t = item_ct1.get_local_id(2) & 15;const int row = item_ct1.get_group(2) * 16 + subwarp; // 0 .. 2*n_ff@@ -2155,6 +2263,46 @@ __dpct_inline__ void native_gu_multi_kernel(}}+// APR (prism.hadamard): y = H_block (signs * x) per block of each row, block = the work-group size (a power of two <= 1024),+// one work-group per (row block, row). The butterflies are the CPU's (hadamard.cpp) in the same order, so the+// rotated values are the same floats.+template <typename T> __dpct_inline__ float ld_f(const T* p) { return (float) *p; }+template <> __dpct_inline__ float ld_f<sycl::half>(const sycl::half* p) { return (float) *p; }+template <typename T> __dpct_inline__ void st_f(T* p, float v) { *p = v; }+template <> __dpct_inline__ void st_f<sycl::half>(sycl::half* p, float v) {+ *p = sycl::vec<float, 1>(v).convert<sycl::half, sycl::rounding_mode::automatic>()[0];+}+template <typename T>+__dpct_inline__ void hadamard_rows_kernel(const T* __restrict__ x, T* __restrict__ y, int64_t width,+ const float* __restrict__ signs, float scale) {+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();+ auto& hbuf = *sycl::ext::oneapi::group_local_memory_for_overwrite<float[1024]>(item_ct1.get_group());+ const int n = (int) item_ct1.get_local_range(2), i = (int) item_ct1.get_local_id(2);+ const int64_t col = (int64_t) item_ct1.get_group(2) * n + i, at = (int64_t) item_ct1.get_group(1) * width + col;+ float v = ld_f<T>(x + at);+ if (signs) v *= signs[col];+ hbuf[i] = v;+ for (int len = 1; len < n; len <<= 1) {+ sycl::group_barrier(item_ct1.get_group());+ if ((i & len) == 0) {+ const float a = hbuf[i], c = hbuf[i + len];+ hbuf[i] = a + c;+ hbuf[i + len] = a - c;+ }+ }+ sycl::group_barrier(item_ct1.get_group());+ st_f<T>(y + at, hbuf[i] * scale);+}+template <typename T> class hadamard_rows_name;+template <typename T>+void launch_hadamard_rows(const T* x, T* y, int64_t rows, int64_t width, int block, const float* signs, dpct::queue_ptr s) {+ const float scale = 1.0f / std::sqrt((float) block);+ s->parallel_for<hadamard_rows_name<T>>(+ sycl::nd_range<3>(sycl::range<3>(1, (size_t) rows, (size_t) (width / block) * (size_t) block),+ sycl::range<3>(1, 1, (size_t) block)),+ [=](sycl::nd_item<3>) [[sycl::reqd_sub_group_size(32)]] { hadamard_rows_kernel<T>(x, y, width, signs, scale); });+}+__dpct_inline__ void swiglu_entries_kernel(const float *__restrict__ gate,const float *__restrict__ up,float *__restrict__ h, long long n) {@@ -2921,6 +3069,29 @@ inline void dq_bf16(const void *vx, int64_t ibs, dst_t *yy, int tid) {cvt<dst_t>(sycl::bit_cast<float>((uint32_t)x[j] << 16));}+// APR trellis types: one 256-value call = two 128-blocks; thread tid decodes block tid/16, lane (tid/2)%8, steps+// 2(tid%2)..2(tid%2)+1 (8 values)+template <int TY, typename dst_t>+inline void dq_tq(const void *vx, int64_t ibs, dst_t *yy, int tid) {+ const int blk = tid >> 4, l = (tid >> 1) & 7, half = tid & 1;+ const auto* b = (const typename TqInfo<TY>::block_t*) vx + 2 * ibs + blk;+ const float d = tq_h2f(b->d);+ uint32_t s[4];+ tq_states4<TY>(b->qs, l, s);+ dst_t* y = yy + blk * 128 + 16 * l + 8 * half;+ float v[8];+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ float w[4];+ tq_step(s[2 * half + i], w);+ v[4 * i + 0] = d * w[0];+ v[4 * i + 1] = d * w[1];+ v[4 * i + 2] = d * w[2];+ v[4 * i + 3] = d * w[3];+ }+ store_run<dst_t, 8>(y, v);+}+// Every type below must also be in is_iq() (BF16: embed_type_supported): the host entry points refuse the others,// so the default is unreachable.template <typename dst_t>@@ -2951,6 +3122,9 @@ dq_dispatch(int ty, const void *vx, int64_t ibs, dst_t *y, int tid) {case 3: dq_q4_1(vx, ibs, y, tid); break;case 8: dq_q8_0(vx, ibs, y, tid); break;case 30: dq_bf16(vx, ibs, y, tid); break;+ case 144: dq_tq<144>(vx, ibs, y, tid); break;+ case 145: dq_tq<145>(vx, ibs, y, tid); break;+ case 146: dq_tq<146>(vx, ibs, y, tid); break;default: break;}}@@ -2978,7 +3152,7 @@ __dpct_inline__ void dequant_gu_kernel(// the types dq_dispatch dequantizesbool is_iq(int t) {- return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||+ return t == 144 || t == 145 || t == 146 || t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||#ifdef STRATA_Q6K_EXPERTSt == 12 || t == 13 || t == 14 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8;#else@@ -4454,6 +4628,58 @@ void iq_set_old_kernels(bool old) { g_old_kernels = old; }void native_expert_set_kq_split(bool on) { g_kq_split = on; }bool iq_old_kernels() { return g_old_kernels; }+// APR (prism.hadamard): see iq_kernels.hpp. A block that is not a power of two in [2, 1024] dividing the row width exits with a+// message, like check() does for a failed launch.+namespace {+void hadamard_bad_block(const char* what, int block, int64_t width) {+ std::fprintf(stderr, "%s: Hadamard block %d is not a power of two in [2, 1024] dividing the row width %lld "+ "(hadamard.txt's block and the expert widths disagree)\n", what, block, (long long) width);+ std::exit(1);+}+constexpr int64_t kHadamardMaxRows = 65535; // the CUDA grid limit; a prompt chunk's rows go up in the same slices+bool hadamard_block_ok(int block, int64_t width) {+ return block >= 2 && block <= 1024 && (block & (block - 1)) == 0 && width % block == 0;+}+} // namespace++void hadamard_rows(const float* x, float* y, int64_t rows, int64_t width, int block, const float* signs, void* stream) {+ if (rows <= 0) return;+ if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows", block, width);+ dpct::queue_ptr s = strata::q_of(stream);+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) {+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);+ launch_hadamard_rows<float>(x + r0 * width, y + r0 * width, n, width, block, signs, s);+ }+ check("hadamard_rows");+}++void hadamard_rows_f16(uint16_t* x, int64_t rows, int64_t width, int block, const float* signs, void* stream) {+ if (rows <= 0) return;+ if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows_f16", block, width);+ dpct::queue_ptr s = strata::q_of(stream);+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) {+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);+ launch_hadamard_rows<sycl::half>((const sycl::half*) x + r0 * width, (sycl::half*) x + r0 * width, n, width, block,+ signs, s);+ }+ check("hadamard_rows_f16");+}++const float* hadamard_device_signs(const float* host, int width) {+ if (host == nullptr || width <= 0) return nullptr;+ static std::mutex mu;+ static std::map<std::tuple<int, const float*, int>, float*> table; // (device, host vector, width)+ const int dev = dpct::get_current_device_id();+ std::lock_guard<std::mutex> lock(mu);+ float*& d = table[std::make_tuple(dev, host, width)];+ if (d == nullptr) {+ sycl::queue& q = dpct::get_in_order_queue();+ d = (float*) strata::malloc_device_guarded((size_t) width * sizeof(float), q, "Hadamard signs");+ q.memcpy(d, host, (size_t) width * sizeof(float)).wait();+ }+ return d;+}+bool iq_supported(int t) noexcept { return is_iq(t); }bool embed_type_supported(int t) noexcept { return is_iq(t) || t == 30; }@@ -4477,6 +4703,9 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {case 2: return (size_t) (n / 32) * sizeof(block_q4_0);case 3: return (size_t) (n / 32) * sizeof(block_q4_1);case 8: return (size_t) (n / 32) * sizeof(block_q8_0);+ case 144: return (size_t) (n / 128) * sizeof(block_tq2_t); // APR trellis types+ case 145: return (size_t) (n / 128) * sizeof(block_tqk6);+ case 146: return (size_t) (n / 128) * sizeof(block_tqk7);case 30: return (size_t) n * 2; // BF16: the token embedding only (iq_embed_rows, iq_dequant_f32)default: return 0;}@@ -5462,7 +5691,8 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long#elseconst bool sw_v1 = v1;#endif- if (sw_v1) {+ // APR: Hadamard-folded down weights take the rotated h, so h exists as floats first (the separate passes)+ if (sw_v1 || L.had_block > 0) {{auto exp_props = sycl::ext::oneapi::experimental::properties{sycl::ext::oneapi::experimental::use_root_sync};@@ -5477,6 +5707,8 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long longswiglu_entries_kernel(gate, up, h, nh);});}+ if (L.had_block > 0) // APR: the down weights are Hadamard-folded, so is their input (in place, every entry row)+ hadamard_rows(h, h, cap_entries, L.n_ff, L.had_block, L.d_signs, stream);{auto exp_props = sycl::ext::oneapi::experimental::properties{sycl::ext::oneapi::experimental::use_root_sync};diff --git a/sycl/src/kernels/tq_sycl_parity.cpp b/sycl/src/kernels/tq_sycl_parity.cppnew file mode 100644index 000000000..c6a161bc5--- /dev/null+++ b/sycl/src/kernels/tq_sycl_parity.cpp@@ -0,0 +1,366 @@+// sycl/src/kernels/tq_sycl_parity.cpp - the Gyro trellis experts (TQ2_T 144, TQK6 145, TQK7 146, Hadamard-folded) on the+// SYCL kernels, against real tensors of a Gyro GGUF.+//+// tq_sycl_parity <pack-dir> <model.gguf> [layer ...]+// tq_sycl_parity --synthetic [--hadamard] GU/DOWN ... (random trellis codes and scales, e.g. 145/146 144/144)+//+// For one expert of each layer (the gate/up/down rows read from the GGUF, the Hadamard signs and block from the pack):+// (a) float reference: fp16-codebook weights, x and h rotated in double, SwiGLU in double.+// (b) the CPU engine path (native_quant_act, native_gu_rows, native_quant_h, native_down_rows).+// (c) the GPU path (hadamard_rows, quantize_q8_1_rows, native_expert_grouped) with 3 entries in one group.+// Checks: (c) against (a) and (b) within the bound every expert type gets (relative L1 < 3e-2); the GPU Hadamard rotation+// against the CPU's; the gate/up rows and the down rows of (c) BIT FOR BIT against a host emulation of the kernels'+// integer and float operations (the int8 codebook, the q8_1 blocks and the h the device produced are read back, the+// lane-strided sums and the xor tree are replayed); the prompt-path dequantizers (iq_dequant_f32 / _f16) against the+// fp16 codebook, bit for bit. Exit 0 only if all hold.+#include "strata/artifact/gguf_reader.hpp"+#include "strata/kernels/cpu/native_expert.hpp"+#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"+#include "strata/kernels/cpu/tq_cpu.hpp"+#include "strata/kernels/iq_kernels.hpp"+#include "ggml-common-tq.h"+#include "../../../src/kernels/cpu/tq_states.hpp"+#include "ggml.h"++#include <sycl/sycl.hpp>+#include <dpct/dpct.hpp>++#include <algorithm>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <vector>++#define __device__+#include "../../../src/kernels/cuda/tq_lut_i8.cuh"+#undef __device__++namespace cpu = strata::kernels::cpu;+namespace {+constexpr int NT = 3;+constexpr int64_t H = 2560, FF = 640;+constexpr int LN = 8; // STRATA_EXPERT_LANES++double rel(const std::vector<float>& a, const std::vector<float>& b) {+ double n = 0, d = 0;+ for (size_t i = 0; i < a.size(); ++i) { n += std::fabs((double) a[i] - b[i]); d += std::fabs((double) b[i]); }+ return n / (d + 1e-30);+}+float h2f(uint16_t b) { return cpu::tq_detail::tq_half_to_float(b); }+int kbits(int t) { return t == 144 ? 8 : t == 145 ? 6 : 7; }+size_t bsize(int t) { return 2 + 4 * (size_t) kbits(t); }++// the 32 trellis states of block b (qs after the fp16 scale)+void states(int t, const uint8_t* blk, uint32_t* st) {+ if (t == 144) cpu::tq_detail::tq_states<8>(blk + 2, st);+ else if (t == 145) cpu::tq_detail::tq_states<6>(blk + 2, st);+ else cpu::tq_detail::tq_states<7>(blk + 2, st);+}+// float dequantization of one row of n values with the fp16 codebook+void dq_row(int t, const uint8_t* row, int64_t n, float* y) {+ for (int64_t b = 0; b < n / 128; ++b) {+ const uint8_t* blk = row + b * bsize(t);+ const float d = h2f((uint16_t) (blk[0] | (blk[1] << 8)));+ uint32_t st[32];+ states(t, blk, st);+ for (int s = 0; s < 32; ++s) {+ const uint32_t x = (st[s] & TQ_STATE_MASK) * 0x9e3779b1u;+ const uint32_t pa = x >> 21, pb = (x >> 10) & 2047u;+ y[b * 128 + 4 * s + 0] = d * h2f(tq_lut_f16[2 * pa]);+ y[b * 128 + 4 * s + 1] = d * h2f(tq_lut_f16[2 * pa + 1]);+ y[b * 128 + 4 * s + 2] = d * h2f(tq_lut_f16[2 * pb]);+ y[b * 128 + 4 * s + 3] = d * h2f(tq_lut_f16[2 * pb + 1]);+ }+ }+}+// host replay of the kernels' dot of one row against one q8_1 row (36-byte blocks): lane `sub` of LN accumulates calls+// sub, sub+LN, ... in order, then the xor tree (o = LN/2 .. 1)+float emu_dot(int t, const uint8_t* row, const uint8_t* q8, int64_t n) {+ const int nb = (int) (n / 128);+ float s[LN] = {};+ for (int sub = 0; sub < LN; ++sub)+ for (int k = sub; k < nb * 8; k += LN) {+ const int kbx = k / 8, lane = k % 8;+ const uint8_t* blk = row + (size_t) kbx * bsize(t);+ uint32_t st[32];+ states(t, blk, st);+ int sumi = 0;+ const uint8_t* b8 = q8 + (size_t) (kbx * 4 + lane / 2) * 36;+ for (int i = 0; i < 4; ++i) {+ const uint32_t x = (st[4 * lane + i] & TQ_STATE_MASK) * 0x9e3779b1u;+ const uint16_t p0 = tq_lut_i8[x >> 21], p1 = tq_lut_i8[(x >> 10) & 2047u];+ const int8_t w[4] = {(int8_t) (p0 & 0xff), (int8_t) (p0 >> 8), (int8_t) (p1 & 0xff), (int8_t) (p1 >> 8)};+ const int8_t* a = (const int8_t*) (b8 + 4 + 16 * (lane & 1) + 4 * i);+ for (int j = 0; j < 4; ++j) sumi += (int) w[j] * (int) a[j];+ }+ const uint16_t dsb = (uint16_t) (b8[0] | (b8[1] << 8));+ const float d = h2f((uint16_t) (blk[0] | (blk[1] << 8)));+ volatile float tmp = h2f(dsb) * TQ_LUT_I8_SCALE;+ volatile float v = d * tmp;+ v = v * (float) sumi;+ volatile float acc = s[sub];+ acc = acc + v;+ s[sub] = acc;+ }+ for (int o = LN / 2; o > 0; o >>= 1) {+ float nx[LN];+ for (int l = 0; l < LN; ++l) nx[l] = s[l] + s[l ^ o];+ std::memcpy(s, nx, sizeof s);+ }+ return s[0];+}++template <class T> T* dmalloc(size_t n) { return (T*) sycl::malloc_device((n ? n : 1) * sizeof(T), dpct::get_in_order_queue()); }+template <class T> void to_dev(T* d, const void* h, size_t n) { dpct::get_in_order_queue().memcpy(d, h, n).wait(); }+template <class T> void from_dev(void* h, const T* d, size_t n) { dpct::get_in_order_queue().memcpy(h, d, n).wait(); }++int check_layer(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int seed, const std::string& label) {+ int failures = 0;+ sycl::queue* s = &dpct::get_in_order_queue();+ // (a) float reference (rows dequantized with the fp16 codebook)+ std::vector<float> G((size_t) FF * H), U((size_t) FF * H), D((size_t) H * FF);+ for (int64_t r = 0; r < FF; ++r) {+ dq_row(f.gu_type, blob.data() + r * f.gu_row, H, G.data() + r * H);+ dq_row(f.gu_type, blob.data() + f.up_off + r * f.gu_row, H, U.data() + r * H);+ }+ for (int64_t r = 0; r < H; ++r) dq_row(f.d_type, blob.data() + f.down_off + r * f.d_row, FF, D.data() + r * FF);+ std::mt19937 rng(11 + seed);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::vector<float> x((size_t) NT * H);+ for (auto& v : x) v = nd(rng);+ std::vector<float> xr(x);+ if (f.had_gu) for (int k = 0; k < NT; ++k) cpu::hadamard_rotate(xr.data() + k * H, (int) H, f.had_block, f.gu_signs);+ std::vector<float> ref((size_t) NT * H), got_c((size_t) NT * H), got_g((size_t) NT * H);+ for (int k = 0; k < NT; ++k) {+ std::vector<float> h(FF);+ for (int64_t r = 0; r < FF; ++r) {+ double g = 0, u = 0;+ for (int64_t i = 0; i < H; ++i) { g += (double) G[r * H + i] * xr[k * H + i]; u += (double) U[r * H + i] * xr[k * H + i]; }+ h[r] = (float) ((g / (1.0 + std::exp(-g))) * u);+ }+ if (f.had_d) cpu::hadamard_rotate(h.data(), (int) FF, f.had_block, f.d_signs);+ for (int64_t r = 0; r < H; ++r) {+ double o = 0;+ for (int64_t i = 0; i < FF; ++i) o += (double) D[r * FF + i] * h[i];+ ref[k * H + r] = (float) o;+ }+ }+ // (b) the CPU engine path+ {+ std::vector<std::vector<uint8_t>> act(NT, std::vector<uint8_t>(cpu::kNativeActBytes));+ std::vector<std::vector<uint8_t>> hq(NT, std::vector<uint8_t>(cpu::kNativeHBytes));+ std::vector<std::vector<float>> ff(NT, std::vector<float>(FF));+ const void* a[NT];+ float* ffp[NT];+ const void* hp[NT];+ float* op[NT];+ for (int k = 0; k < NT; ++k) {+ cpu::native_quant_act(f, x.data() + k * H, act[k].data());+ a[k] = act[k].data();+ ffp[k] = ff[k].data();+ }+ cpu::native_gu_rows(f, blob.data(), a, NT, ffp, 0, (int) FF);+ for (int k = 0; k < NT; ++k) {+ cpu::native_quant_h(f, ff[k].data(), hq[k].data());+ hp[k] = hq[k].data();+ op[k] = got_c.data() + k * H;+ }+ cpu::native_down_rows(f, blob.data(), hp, NT, op, 0, (int) H);+ }+ // (c) the GPU: one group holding the NT entries+ auto L = strata::kernels::native_expert_layout(f.gu_type, f.d_type, H, FF);+ if (f.had_d) { L.had_block = f.had_block; L.d_signs = strata::kernels::hadamard_device_signs(f.d_signs, (int) FF); }+ uint8_t* dblob = dmalloc<uint8_t>(blob.size());+ float* dx = dmalloc<float>(x.size());+ uint8_t* dxq = dmalloc<uint8_t>((size_t) NT * H / 32 * 36);+ const size_t scr_bytes = strata::kernels::native_expert_scratch_bytes(NT, FF);+ uint8_t* dscr = dmalloc<uint8_t>(scr_bytes);+ float* dout = dmalloc<float>((size_t) NT * H);+ unsigned long long* dptr = dmalloc<unsigned long long>(1);+ int32_t *dstart = dmalloc<int32_t>(2), *dn = dmalloc<int32_t>(1), *ddst = dmalloc<int32_t>(NT), *dtok = dmalloc<int32_t>(NT);+ to_dev(dblob, blob.data(), blob.size());+ to_dev(dx, x.data(), x.size() * 4);+ const unsigned long long p = (unsigned long long) dblob;+ const int32_t st2[2] = {0, NT}, one = 1, idx[NT] = {0, 1, 2};+ to_dev(dptr, &p, 8);+ to_dev(dstart, st2, 8);+ to_dev(dn, &one, 4);+ to_dev(ddst, idx, NT * 4);+ to_dev(dtok, idx, NT * 4);+ if (f.had_gu) {+ strata::kernels::hadamard_rows(dx, dx, NT, H, f.had_block, strata::kernels::hadamard_device_signs(f.gu_signs, (int) H), s);+ std::vector<float> gx(x.size());+ s->wait();+ from_dev(gx.data(), dx, gx.size() * 4);+ const bool same = std::memcmp(gx.data(), xr.data(), gx.size() * 4) == 0;+ std::printf(" gpu hadamard_rows vs cpu hadamard_rotate: %s (rel %.2e)\n", same ? "bitwise equal" : "differ", rel(gx, xr));+ if (!(rel(gx, xr) < 1e-6)) ++failures;+ std::vector<uint16_t> xh(x.size()), xh_out(x.size());+ for (size_t i = 0; i < x.size(); ++i) xh[i] = ggml_fp32_to_fp16(x[i]);+ uint16_t* dxh = dmalloc<uint16_t>(xh.size());+ to_dev(dxh, xh.data(), xh.size() * 2);+ strata::kernels::hadamard_rows_f16(dxh, NT, H, f.had_block, strata::kernels::hadamard_device_signs(f.gu_signs, (int) H), s);+ s->wait();+ from_dev(xh_out.data(), dxh, xh.size() * 2);+ sycl::free(dxh, *s);+ std::vector<float> xf(x.size());+ for (size_t i = 0; i < xf.size(); ++i) xf[i] = ggml_fp16_to_fp32(xh_out[i]);+ std::printf(" gpu hadamard_rows_f16 vs cpu float rotation: rel %.2e\n", rel(xf, xr));+ if (!(rel(xf, xr) < 2e-3)) ++failures;+ }+ strata::kernels::quantize_q8_1_rows(dx, NT, H, dxq, s);+ strata::kernels::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, 1, NT, dxq, dscr, dout, s);+ s->wait();+ from_dev(got_g.data(), dout, got_g.size() * 4);+ { // bit-for-bit replay of the lane kernels+ std::vector<uint8_t> hxq((size_t) NT * H / 32 * 36);+ from_dev(hxq.data(), dxq, hxq.size());+ const size_t fb = (size_t) NT * FF * 4, fa = (fb + 255) & ~(size_t) 255;+ std::vector<float> gate((size_t) NT * FF), up((size_t) NT * FF), hh((size_t) NT * FF);+ from_dev(gate.data(), (const float*) dscr, gate.size() * 4);+ from_dev(up.data(), (const float*) (dscr + fa), up.size() * 4);+ from_dev(hh.data(), (const float*) (dscr + 2 * fa), hh.size() * 4);+ std::vector<uint8_t> hhq((size_t) NT * FF / 32 * 36);+ from_dev(hhq.data(), dscr + 3 * fa, hhq.size());+ size_t gu_diff = 0, dn_diff = 0;+ for (int k = 0; k < NT; ++k) {+ for (int64_t r = 0; r < FF; ++r) {+ const float eg = emu_dot(f.gu_type, blob.data() + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);+ const float eu = emu_dot(f.gu_type, blob.data() + f.up_off + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);+ gu_diff += std::memcmp(&eg, &gate[k * FF + r], 4) != 0;+ gu_diff += std::memcmp(&eu, &up[k * FF + r], 4) != 0;+ }+ for (int64_t r = 0; r < H; ++r) {+ const float ed = emu_dot(f.d_type, blob.data() + f.down_off + r * f.d_row, hhq.data() + (size_t) k * FF / 32 * 36, FF);+ dn_diff += std::memcmp(&ed, &got_g[k * H + r], 4) != 0;+ }+ }+ std::printf(" kernel replay on the host (int8 codebook, q8_1, lane sums): gate/up %zu of %lld, down %zu of %lld values differ in any bit\n",+ gu_diff, (long long) (2 * NT * FF), dn_diff, (long long) (NT * H));+ if (gu_diff || dn_diff) ++failures;+ (void) hh;+ }+ { // (d) the GPU dequantizers (prompt path) against the fp16-codebook floats+ float* dq = dmalloc<float>((size_t) FF * H);+ uint16_t* dh = dmalloc<uint16_t>((size_t) FF * H);+ std::vector<float> got_dq((size_t) FF * H);+ std::vector<uint16_t> got_h((size_t) FF * H);+ size_t differ = 0, h_differ = 0;+ const struct { const uint8_t* src; int type; const std::vector<float>* want; } mats[3] = {+ {dblob, f.gu_type, &G}, {dblob + f.up_off, f.gu_type, &U}, {dblob + f.down_off, f.d_type, &D}};+ for (const auto& m : mats) {+ strata::kernels::iq_dequant_f32(m.type, m.src, FF * H, dq, s);+ strata::kernels::iq_dequant_f16(m.type, m.src, FF * H, dh, s);+ s->wait();+ from_dev(got_dq.data(), dq, got_dq.size() * 4);+ from_dev(got_h.data(), dh, got_h.size() * 2);+ for (size_t i = 0; i < got_dq.size(); ++i) {+ differ += std::memcmp(&got_dq[i], &(*m.want)[i], 4) != 0;+ h_differ += got_h[i] != ggml_fp32_to_fp16(got_dq[i]);+ }+ }+ sycl::free(dq, *s); sycl::free(dh, *s);+ std::printf(" GPU dequant vs fp16-codebook floats: %zu of %lld values differ in any bit; fp16 of it: %zu differ\n",+ differ, (long long) (3 * FF * H), h_differ);+ if (differ || h_differ) ++failures;+ }+ sycl::free(dblob, *s); sycl::free(dx, *s); sycl::free(dxq, *s); sycl::free(dscr, *s); sycl::free(dout, *s);+ sycl::free(dptr, *s); sycl::free(dstart, *s); sycl::free(dn, *s); sycl::free(ddst, *s); sycl::free(dtok, *s);+ const double ec = rel(got_c, ref), eg = rel(got_g, ref), ecg = rel(got_c, got_g);+ const bool ok = ec < 3e-2 && eg < 3e-2 && std::isfinite(ec) && std::isfinite(eg);+ std::printf("%-9s types %d/%d blob %8zu cpu rel %.2e gpu rel %.2e cpu-gpu %.2e %s\n", label.c_str(), f.gu_type, f.d_type,+ f.bytes, ec, eg, ecg, ok ? "ok" : "FAIL");+ return failures + (ok ? 0 : 1);+}+} // namespace++// random trellis bytes (every bit pattern is a valid trellis), scales in a sane fp16 range+std::vector<uint8_t> synthetic_blob(const cpu::NativeFmt& f, int seed) {+ std::mt19937 rng(1234 + seed);+ std::vector<uint8_t> blob(f.bytes);+ for (auto& b : blob) b = (uint8_t) rng();+ auto fix = [&](uint8_t* base, int type, size_t row_bytes, int64_t rows) {+ for (int64_t r = 0; r < rows; ++r)+ for (size_t b = 0; b < row_bytes / bsize(type); ++b) {+ uint8_t* blk = base + r * row_bytes + b * bsize(type);+ const float d = 0.002f + 0.03f * ((rng() & 0xffff) / 65535.0f);+ const uint16_t h = ggml_fp32_to_fp16(d);+ blk[0] = (uint8_t) (h & 0xff);+ blk[1] = (uint8_t) (h >> 8);+ }+ };+ fix(blob.data(), f.gu_type, f.gu_row, FF);+ fix(blob.data() + f.up_off, f.gu_type, f.gu_row, FF);+ fix(blob.data() + f.down_off, f.d_type, f.d_row, H);+ return blob;+}+int type_of(const std::string& s) { return std::atoi(s.c_str()); }++int main(int argc, char** argv) {+ setvbuf(stdout, nullptr, _IONBF, 0);+ if (argc >= 2 && std::string(argv[1]) == "--synthetic") {+ int failures = 0, i = 2;+ const bool had = argc > 2 && std::string(argv[2]) == "--hadamard";+ if (had) ++i;+ static std::vector<float> sx((size_t) H), sh((size_t) FF);+ std::mt19937 srng(5);+ for (auto& v : sx) v = (srng() & 1) ? 1.f : -1.f;+ for (auto& v : sh) v = (srng() & 1) ? 1.f : -1.f;+ for (; i < argc; ++i) {+ const std::string arg = argv[i];+ const size_t slash = arg.find('/');+ const int gu = type_of(arg.substr(0, slash)), dn = type_of(arg.substr(slash + 1));+ cpu::NativeFmt f;+ std::string err;+ if (slash == std::string::npos || !cpu::native_fmt(gu, dn, H, FF, f, err) ||+ !strata::kernels::native_expert_supported(gu, dn, H, FF)) {+ std::printf("%s: %s\n", arg.c_str(), err.empty() ? "not a pair the GPU expert kernels take" : err.c_str());+ ++failures;+ continue;+ }+ if (had) {+ f.had_block = 128;+ f.had_gu = f.had_d = true;+ f.gu_signs = sx.data();+ f.d_signs = sh.data();+ }+ failures += check_layer(f, synthetic_blob(f, i), i, had ? "hadamard" : "synthetic");+ }+ std::printf("tq_sycl_parity: %d failures\n", failures);+ return failures ? 1 : 0;+ }+ if (argc < 3) { std::fprintf(stderr, "usage: tq_sycl_parity <pack-dir> <model.gguf> [layer ...]\n"); return 2; }+ std::string err;+ if (!cpu::expert_layout_load(argv[1], 48, 512, err)) { std::printf("expert_layout_load: %s\n", err.c_str()); return 1; }+ const auto& lay = cpu::expert_layout();+ const strata::GgufModel model(strata::gguf_split_paths(argv[2]));+ std::vector<int> layers;+ for (int i = 3; i < argc; ++i) layers.push_back(std::atoi(argv[i]));+ if (layers.empty()) layers = {0, 1, 2, 3, 20, 47};+ int failures = 0;+ for (int l : layers) {+ const strata::TensorInfo* t[3] = {};+ const uint8_t* data[3] = {};+ const char* roles[3] = {"gate", "up", "down"};+ for (int r = 0; r < 3; ++r) {+ size_t at = 0;+ t[r] = model.find("blk." + std::to_string(l) + ".ffn_" + roles[r] + "_exps.weight", &at);+ if (t[r]) data[r] = model.shard(at).tensor_data(*t[r]);+ }+ if (!t[0] || !t[1] || !t[2]) { std::printf("layer %d: no expert tensors\n", l); ++failures; continue; }+ cpu::NativeFmt f = lay.fmt[(size_t) l];+ for (int E : {7, 100}) {+ std::vector<uint8_t> blob(f.bytes);+ std::memcpy(blob.data(), data[0] + (size_t) E * f.up_off, f.up_off);+ std::memcpy(blob.data() + f.up_off, data[1] + (size_t) E * f.up_off, f.up_off);+ std::memcpy(blob.data() + f.down_off, data[2] + (size_t) E * (f.bytes - f.down_off), f.bytes - f.down_off);+ failures += check_layer(f, blob, l * 1000 + E, "layer " + std::to_string(l) + " e" + std::to_string(E));+ }+ }+ std::printf("tq_sycl_parity: %d failures\n", failures);+ return failures ? 1 : 0;+}diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cppindex 92cedf489..28222d91a 100644--- a/sycl/src/prefill/prefill.cpp+++ b/sycl/src/prefill/prefill.cpp@@ -24,6 +24,7 @@#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/iq_dequant_expert.hpp"#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"#include "strata/kernels/qsa.hpp"#include "strata/kernels/kv_stream.hpp"#include "strata/kernels/cvec.hpp"@@ -715,6 +716,9 @@ struct Prefill::Impl {uint16_t *xn16 = nullptr, *lo16 = nullptr;float* mixed = nullptr;uint16_t *mixed_bf = nullptr, *mixed_h = nullptr;+ // APR (prism.hadamard): per layer, this device's copies of the gate/up (n_embd) and down (n_ff) sign vectors+ // (nullptr: identity signs or not folded), resolved once in init so a prompt chunk reads a pointer+ std::vector<const float*> had_sx, had_sh;bool f16_io = false; // HIP, STRATA_HIP_PROMPT_F16=1: this stage's 16-bit GEMMs run FP16 in and out (set in init)uint16_t *xn16_lo = nullptr, *lo16_lo = nullptr, *mixed_bf_lo = nullptr; // bf16x2(): the BF16 GEMMs' low partsfloat* bo = nullptr;@@ -1187,6 +1191,17 @@ bool Prefill::init(const core::WeightTable &wt, const core::ModelGeometry &g,if (g.n_embd != N || g.hc != HC || g.hc_lr != LR || g.n_expert < 1 || ss.k != K) {err = "prefill: geometry differs from the artifact's"; return false;}+ { // APR: the sign vectors on this device, once (hadamard_device_signs takes a mutex and allocates its copy)+ const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();+ m.had_sx.assign(lay.native ? lay.fmt.size() : 0, nullptr);+ m.had_sh.assign(m.had_sx.size(), nullptr);+ for (size_t l = 0; l < m.had_sx.size(); ++l) {+ if (lay.fmt[l].had_gu)+ m.had_sx[l] = strata::kernels::hadamard_device_signs(lay.fmt[l].gu_signs, (int) lay.fmt[l].n_embd);+ if (lay.fmt[l].had_d)+ m.had_sh[l] = strata::kernels::hadamard_device_signs(lay.fmt[l].d_signs, (int) lay.fmt[l].n_ff);+ }+ }m.device = dpct::get_current_device_id();if (static bool said_i8 = false; !said_i8 && i8_mode(m.cs) != 0) {said_i8 = true;@@ -3471,6 +3486,17 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,// (moe_fused_iq.hpp) where they cover its two formats, else MMQ (or the FP16 path: IQ1_M).const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];+ // APR (prism.hadamard): folded gate/up take H (s * x) on the gathered fp16 rows, folded down H (s * h) on+ // the fp16 SwiGLU output, both rotated in fp16 storage with fp32 arithmetic (as the CUDA FP16 path does).+ // The fused int8 products and the MMQ path do not rotate; they take only formats of unfolded layers.+ const bool had_gu = lay.native && lay.fmt[(size_t) l].had_gu;+ const bool had_d = lay.native && lay.fmt[(size_t) l].had_d;+ const int had_block = lay.native ? lay.fmt[(size_t) l].had_block : 0;+ const float* had_sx = had_gu ? m.had_sx[(size_t) l] : nullptr;+ const float* had_sh = had_d ? m.had_sh[(size_t) l] : nullptr;+ auto had_h = [&](int64_t row0, int64_t nrows) {+ if (had_d) strata::kernels::hadamard_rows_f16(m.Hh + row0 * 640, nrows, 640, had_block, had_sh, m.cs);+ };const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42;const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42;// --peer-device: MMQ only, whether or not the peer took the prompt path (set_peer can decline), as@@ -3715,6 +3741,8 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,}} else {gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs);+ // APR: Xs is this layer's private gathered copy (rotated in fp16 storage, fp32 arithmetic)+ if (had_gu) strata::kernels::hadamard_rows_f16(m.Xs, T * K, N, had_block, had_sx, m.cs);}// multi-GPU: the peer's share, enqueued before the primary's own experts so both cards work at oncepeer_now = use_mmq && !order_peer.empty();@@ -4195,11 +4223,11 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,const int gt = lay.fmt[(size_t) l].gu_type, dt = lay.fmt[(size_t) l].d_type;return (gt == 18 || gt == 21 || gt == 22 || gt == 23) && (dt == 20 || dt == 42);};- const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt();+ const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt() && !had_d;// 151-b70: the fused int8 DPAS products (IQ3_S gate/up, IQ4_NL down, the 640 x 2560 expert), the same// held group of staged experts as aboveconst int i8m = i8_mode(m.cs);- const bool i8on = !xmx2_only && i8m != 0 && !use_mmq && lay.native &&+ const bool i8on = !xmx2_only && i8m != 0 && !use_mmq && lay.native && !had_gu && !had_d &&(lay.fmt[(size_t) l].gu_type == 21 || lay.fmt[(size_t) l].gu_type == 18 || lay.fmt[(size_t) l].gu_type == 22) &&(lay.fmt[(size_t) l].d_type == 20 || lay.fmt[(size_t) l].d_type == 42) &&lay.fmt[(size_t) l].n_ff == 640 && lay.fmt[(size_t) l].n_embd == 2560 &&@@ -4293,6 +4321,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,pt.mark(kPfGemmGU, cs);m.gemm.f16(m.Xs + o * N, m.dq_gu[qd], m.GU + o * 1280, xg_c[i], 1280, N);swiglu_interleaved(m.GU + o * 1280, m.Hh + o * 640, xg_c[i], m.cs);+ had_h(o, xg_c[i]);pt.mark(kPfGemmD, cs);m.gemm.f16(m.Hh + o * 640, m.dq_d[qd], m.Dm + o * N, xg_c[i], N, 640);o += xg_c[i];@@ -4322,6 +4351,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,strata::kernels::iq_xmx_grouped(f.gu_type, xg_g, xg_u, xg_c, xg_n, m.Xs + xg_r0 * N,m.GU + xg_r0 * 1280, 1280, (int) N, m.cs);swiglu_interleaved(m.GU + xg_r0 * 1280, m.Hh + xg_r0 * 640, nr, m.cs);+ had_h(xg_r0, nr);pt.mark(kPfGemmD, cs);strata::kernels::iq_xmx_grouped(f.d_type, xg_d, nullptr, xg_c, xg_n, m.Hh + xg_r0 * 640,m.Dm + xg_r0 * N, (int) N, 640, m.cs);@@ -4336,6 +4366,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,pt.mark(kPfGemmGU, cs);m.gemm.f16(m.Xs + o * N, m.dq_gu[q], m.GU + o * 1280, xg_c[i], 1280, N);swiglu_interleaved(m.GU + o * 1280, m.Hh + o * 640, xg_c[i], m.cs);+ had_h(o, xg_c[i]);pt.mark(kPfGemmD, cs);m.gemm.f16(m.Hh + o * 640, m.dq_d[q], m.Dm + o * N, xg_c[i], N, 640);o += xg_c[i];@@ -4473,6 +4504,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,}}swiglu_interleaved(m.GU + r0 * 1280, m.Hh + r0 * 640, nr, m.cs);+ had_h(r0, nr);pt.mark(kPfGemmD, cs);strata::prefill::xmx::grouped_f16(m.Hh + r0 * 640, wd, gc, ng, m.Dm + r0 * N, (int) N, 640,m.cs);@@ -4482,6 +4514,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,pt.mark(kPfGemmGU, cs);m.gemm.f16(m.Xs + o0 * N, m.dq_gu[q], m.GU + o0 * 1280, ne, 1280, N);swiglu_interleaved(m.GU + o0 * 1280, m.Hh + o0 * 640, ne, m.cs);+ had_h(o0, ne);pt.mark(kPfGemmD, cs);m.gemm.f16(m.Hh + o0 * 640, m.dq_d[q], m.Dm + o0 * N, ne, N, 640);return true;