Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: trellis routed experts end to end on the device and the CPU lane (TQK6/7, TQK5X/4X computed, TQK5T/4T table): ds4_tq.cu (gfx12 WMMA + portable FMA tiles over the file layout), ds4_tq_cpu.cpp (Strata tq_cpu kernels), iq entry points route the types; test ds4_tq_test (all six types pass: device and CPU lane within 1e-6 of a double reference)
Niko1221e5b4021c6f0d
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 5ea05b40b..e3478900b 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -776,12 +776,30 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_include_directories(strata_dsv41_kernels PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cu- src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp)+ src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp+ src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})find_package(Threads REQUIRED)target_link_libraries(strata-dsv41 PRIVATE Threads::Threads)+ if(STRATA_NATIVE_EXPERTS) # Strata's CPU trellis kernels (tq_cpu.hpp) for the trellis experts on the CPU lane+ target_link_libraries(strata-dsv41 PRIVATE strata_kernels_cpu)+ target_compile_definitions(strata-dsv41 PRIVATE STRATA_DSV41_TQ_CPU=1)+ endif()+ if(STRATA_NATIVE_EXPERTS)+ # the trellis experts (device kernels ds4_tq.cu, CPU lane ds4_tq_cpu.cpp) against a double reference; needs a GPU, no model+ set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE HIP)+ add_executable(ds4_tq_test tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp+ src/dsv41/ds4_iq_cpu.cpp tools/trellis/tqenc.cpp)+ target_include_directories(ds4_tq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}+ ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)+ target_link_libraries(ds4_tq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)+ target_compile_definitions(ds4_tq_test PRIVATE STRATA_DSV41_TQ_CPU=1)+ if(STRATA_ENABLE_HIP)+ target_compile_options(ds4_tq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")+ endif()+ endif()if(STRATA_ENABLE_HIP)foreach(_t strata_dsv41_kernels strata-dsv41)target_compile_options(${_t} PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hppindex ac6d6f176..7555fccfb 100644--- a/include/strata/kernels/cpu/tq_cpu.hpp+++ b/include/strata/kernels/cpu/tq_cpu.hpp@@ -74,7 +74,7 @@ void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, si/// Largest token group per call and widest row the kernels take.inline constexpr int kTqMaxTokens = 8;-inline constexpr int kTqMaxN = 4096;+inline constexpr int kTqMaxN = 5120; // (DeepSeek-V4.1 hidden width; was 4096)namespace tq_detail {/// One token's Q8_0 activation unpacked for the kernels: the int8 values widened to int16 and the scales as floats.diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 556d8c1bc..9b8e87014 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1002,9 +1002,9 @@ int launch_errors() { return g_errors.load(std::memory_order_relaxed); }size_t type_row_bytes(int type, int64_t k) {if (type == kTypeMXFP4) return k % kMxBlock == 0 ? mx_row_bytes(k) : 0;- return iq::supported(type) && k % 256 == 0 ? iq::row_bytes(type, k) : 0;+ return iq::supported(type) && k % iq::block_values(type) == 0 ? iq::row_bytes(type, k) : 0;}-int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? 256 : 0; }+int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? iq::block_values(type) : 0; }const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }namespace {@@ -1024,8 +1024,8 @@ bool iq_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit, iq:ia.limit = limit;ia.tg = expert_type(lay, 0);ia.td = expert_type(lay, 2);- if (ia.tg != expert_type(lay, 1) || !iq::supported(ia.tg) || !iq::supported(ia.td) || n_embd % 256 != 0 ||- n_ff % 256 != 0) {+ if (ia.tg != expert_type(lay, 1) || !iq::supported(ia.tg) || !iq::supported(ia.td) || n_embd % iq::block_values(ia.tg) != 0 ||+ n_ff % iq::block_values(ia.td) != 0) {std::fprintf(stderr, "ds4 gguf %s: expert types %s / %s / %s (n_embd %d, n_ff %d) not covered\n", who,type_name(expert_type(lay, 0)), type_name(expert_type(lay, 1)), type_name(expert_type(lay, 2)),n_embd, n_ff);diff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hppindex 32b40c70a..286afd2ea 100644--- a/src/dsv41/ds4_gguf_cpu.hpp+++ b/src/dsv41/ds4_gguf_cpu.hpp@@ -36,6 +36,16 @@ void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, consfloat* const* ff, int r0, int r1, float limit);/// out[t][r] = down_r . hq[t] for rows [r0, r1).void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);+/// The trellis types (ds4_iq.cuh's is_tq; ds4_tq_cpu.cpp, Strata's tq_cpu kernels): the same four calls, reached from the+/// iq_cpu_* ones above for those types (their activations: F32 scale a 32 values, then the int8 values).+bool tq_cpu_type(int type);+int tq_cpu_act();+bool tq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);+void tq_cpu_quant(const float* x, int k, void* dst);+void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,+ int r0, int r1, float limit);+void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);+/// The CPU lane's MXFP4 rows in the device's arithmetic (ds4_gguf.cu's mx_dot): the activations rounded to int8 per 32/// values as the device rounds them (q8_round's q and scale), each block's products summed exactly in integers, then/// times (weight scale x activation scale) into F32 sums - the same numbers as the device up to the order of thediff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex ec2c70819..ba62fccce 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -910,7 +910,10 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {bool supported(int t) { return block_bytes(t) > 0; }+int block_values(int t) { return is_tq(t) ? 128 : block_bytes(t) > 0 ? 256 : 0; }+int block_bytes(int t) {+ if (is_tq(t)) return 2 + 4 * tq_kbits(t);switch (t) {case kIQ2_XXS: return Ty<kIQ2_XXS>::B;case kIQ2_S: return Ty<kIQ2_S>::B;@@ -926,6 +929,12 @@ int block_bytes(int t) {const char* name(int t) {switch (t) {+ case kTQK6: return "TQK6";+ case kTQK7: return "TQK7";+ case kTQK5X: return "TQK5X";+ case kTQK4X: return "TQK4X";+ case kTQK5T: return "TQK5T";+ case kTQK4T: return "TQK4T";case kIQ2_XXS: return "IQ2_XXS";case kIQ2_S: return "IQ2_S";case kIQ3_XXS: return "IQ3_XXS";@@ -939,6 +948,7 @@ const char* name(int t) {}bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {+ if (is_tq(a.tg)) return tq::gate_up(a, groups, ne_max, s);if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);#define GU_(T) gu_ne<T>(a, grid, ne_max, s)@@ -948,6 +958,7 @@ bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {}bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {+ if (is_tq(a.td)) return tq::down(a, groups, ne_max, s);if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);#define DN_(T) dn_ne<T>(a, grid, ne_max, s)@@ -957,15 +968,18 @@ bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {}bool light_gate_up(const SetArgs& a, cudaStream_t s) {+ if (is_tq(a.tg)) return false; // (the trellis sets take the tiles for every expert)if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;return light_set(a, true, s);}bool light_down(const SetArgs& a, cudaStream_t s) {+ if (is_tq(a.td)) return false;if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;return light_set(a, false, s);}bool set_gate_up(const SetArgs& a, cudaStream_t s) {+ if (is_tq(a.tg)) return tq::set_gate_up(a, s);if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;SetArgs t = a;if (light_gate_up(a, s)) { // the experts of a few rows; the tiles the others@@ -980,6 +994,7 @@ bool set_gate_up(const SetArgs& a, cudaStream_t s) {}bool set_down(const SetArgs& a, cudaStream_t s) {+ if (is_tq(a.td)) return tq::set_down(a, s);if (a.n_ff % 256 != 0 || a.n_embd % BN != 0) return false;SetArgs t = a;if (light_down(a, s)) {@@ -994,6 +1009,17 @@ bool set_down(const SetArgs& a, cudaStream_t s) {}bool mv(const MvJob* jobs, int n, cudaStream_t s) {+ { // the trellis jobs go to their kernels, the others on+ MvJob tqj[64], oth[64];+ int nt = 0, no = 0;+ if (n > 64) return false;+ for (int i = 0; i < n; ++i) (is_tq(jobs[i].type) ? tqj[nt++] : oth[no++]) = jobs[i];+ if (nt > 0) {+ if (!tq::mv(tqj, nt, s)) return false;+ if (no == 0) return true;+ return mv(oth, no, s);+ }+ }// the jobs by type, up to kMaxJobs a launchbool done[64] = {};if (n > 64) return false;diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 65711f517..f13199fdc 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -22,12 +22,23 @@ namespace strata::ds4::gguf::iq {constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,kIQ4_XS = 23;+/// The trellis types (Strata's, ds4_tq.cu): ggml's TQK6 / TQK7 (145 / 146, the table codebook) and the experimental+/// TQK5X / TQK4X (224 / 225, computed codebook) and TQK5T / TQK4T (226 / 227, table): a block of 128 values is+/// {fp16 d; 32 steps of K bits}, 2 + 4 K bytes (tools/trellis/tqenc.hpp). They go through the same entry points below.+constexpr int kTQK6 = 145, kTQK7 = 146, kTQK5X = 224, kTQK4X = 225, kTQK5T = 226, kTQK4T = 227;+constexpr int tq_kbits(int type) {+ return type == kTQK6 ? 6 : type == kTQK7 ? 7 : (type == kTQK5X || type == kTQK5T) ? 5 : (type == kTQK4X || type == kTQK4T) ? 4 : 0;+}+constexpr bool is_tq(int type) { return tq_kbits(type) > 0; }+/// Whether the device kernels take this ggml type.bool supported(int type);-/// Bytes of a block (256 values) of the type (0: not one of these).+/// Values a block of the type holds (256; the trellis types 128; 0: not one of these).+int block_values(int type);+/// Bytes of a block of the type (0: not one of these).int block_bytes(int type);-/// Bytes of a row of k values (k % 256 == 0).-inline size_t row_bytes(int type, int64_t k) { return (size_t) (k / 256) * (size_t) block_bytes(type); }+/// Bytes of a row of k values (k % block_values == 0).+inline size_t row_bytes(int type, int64_t k) { return (size_t) (k / block_values(type)) * (size_t) block_bytes(type); }/// The type's name ("?" when unknown).const char* name(int type);@@ -101,4 +112,14 @@ struct MvJob {};bool mv(const MvJob* jobs, int n, cudaStream_t s);+/// The trellis types' kernels (ds4_tq.cu), reached from the entry points above for those types: arithmetic as the CPU lane's+/// (ds4_tq_cpu.cpp: int8 activations with an F32 scale a 32, exact weights), on the tensor cores of gfx12 (WMMA) or in FMAs.+namespace tq {+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s);+bool down(const Args& a, int groups, int ne_max, cudaStream_t s);+bool set_gate_up(const SetArgs& a, cudaStream_t s);+bool set_down(const SetArgs& a, cudaStream_t s);+bool mv(const MvJob* jobs, int n, cudaStream_t s);+} // namespace tq+} // namespace strata::ds4::gguf::iqdiff --git a/src/dsv41/ds4_iq_cpu.cpp b/src/dsv41/ds4_iq_cpu.cppindex f96c81952..e0f04678a 100644--- a/src/dsv41/ds4_iq_cpu.cpp+++ b/src/dsv41/ds4_iq_cpu.cpp@@ -756,14 +756,22 @@ void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, intbool iq_cpu_own_rows(int type) { return own(type); }#else-bool iq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {- err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts' CPU lane is not in it";+// (no ggml-cpu lane for the low-bit types here; the trellis types have their own: ds4_tq_cpu.cpp)+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {+ if (tq_cpu_type(tg) && tq_cpu_type(td)) return tq_cpu_fmt(tg, td, n_embd, n_ff, f, err);+ err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts CPU lane is not in it";return false;}-void iq_cpu_quant(int, const float*, int, void*) {}-void iq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int,- float) {}-void iq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}+void iq_cpu_quant(int act, const float* x, int k, void* dst) {+ if (act == tq_cpu_act()) tq_cpu_quant(x, k, dst);+}+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,+ int r0, int r1, float limit) {+ if (tq_cpu_type(f.tg)) tq_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);+}+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {+ if (tq_cpu_type(f.td)) tq_cpu_down(f, down, hq, nt, out, r0, r1);+}bool iq_cpu_own_rows(int) { return false; }#endifdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 369b51387..f30a982eb 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1478,7 +1478,7 @@ struct Model::Impl {// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)static int gg_type_of(const std::string& dtype) {- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23})+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})if (dtype == gg::type_name(t)) return t;return -1;}diff --git a/src/dsv41/ds4_tq.cu b/src/dsv41/ds4_tq.cunew file mode 100644index 000000000..63883c646--- /dev/null+++ b/src/dsv41/ds4_tq.cu@@ -0,0 +1,465 @@+// Strata. The trellis routed experts (TQK6 / TQK7 / TQK5X / TQK4X / TQK5T / TQK4T, ds4_iq.cuh's is_tq) on the device for the+// DeepSeek-V4.1 engine path. The weights stay as the file has them (a row of k values = k / 128 blocks of {fp16 d; 4 K stream+// bytes}); the blob in VRAM is the file's bytes, so the expert cache, the tensor-parallel cuts and the CPU lane read the same+// layout. Arithmetic as the CPU lane's (ds4_tq_cpu.cpp): the activations are int8 with an F32 scale per 32 values (the+// engine's q8_quant), the weights exact (fp16 codebook values times the block's fp16 scale); the sums are F32.+//+// gfx12 (RDNA4, R9700): v_wmma_f32_16x16x16_f16. The weights are the B operand (16 rows of a tile = the 16 columns, a lane+// holds one row's 8 consecutive k), the activations the A operand (16 entries = the 16 rows, split x = hi + lo in two fp16+// so the product keeps ~2^-22), D[entry][weight row] in F32: a lane holds the entries 8 kg .. 8 kg + 7 of its row, so the+// block scale d is the lane's own. Lane l = r + 16 kg owns row r and the 16 steps 16 kg .. 16 kg + 15 of every block, whose+// 15-bit states are one 128-bit window of the stream (the tile code of tools/dsv41_lab/wmma_emu.hpp, checked there against a+// double reference and on the card by tests/dsv41/tqw_hip_test.cpp). Elsewhere the same lane mapping in plain FMAs.+#include "ds4_iq.cuh"++#include <cuda_fp16.h>++#include "ggml-common-tq.h" // tq_lut_f16 (the 2048-point codebook as fp16 bit patterns, __device__)++#include <algorithm>+#include <cstdint>+#include <cstdio>+#include <cstdlib>++#if defined(STRATA_USE_HIP) && (defined(__GFX12__) || defined(__gfx1200__) || defined(__gfx1201__))+#define DS4_TQ_WMMA 1+#endif++namespace strata::ds4::gguf::iq::tq {+namespace {++constexpr int kWaves = 4, kThreads = kWaves * 32; // a block: 4 waves, a wave = one 16-row tile of each matrix+constexpr int kTileRows = 16;+constexpr int kMaxEnt = 16; // entries (columns) a pass++#if defined(DS4_TQ_WMMA)+typedef _Float16 half8_t __attribute__((ext_vector_type(8)));+typedef float float8_t __attribute__((ext_vector_type(8)));+#endif++constexpr bool computed_code(int type) { return type == kTQK5X || type == kTQK4X; }++// ---------------------------------------------------------------- the codebook and the stream+// a step's 15-bit state -> the four weights as two words of two fp16 (lo, hi): w0 w1 | w2 w3+template <bool COMPUTED>+__device__ __forceinline__ void step_words(const uint32_t* lut, uint32_t s, uint32_t& w0, uint32_t& w1) {+ if (COMPUTED) {+ w0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;+ w1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;+ } else {+ const uint32_t x = s * 0x9e3779b1u;+ w0 = lut[x >> 21];+ w1 = lut[(x >> 10) & 2047u];+ }+}+__device__ __forceinline__ float half_bits_f(uint32_t b) { return __half2float(__ushort_as_half((unsigned short) b)); }++// the 128-bit window of a lane's 16 steps in one block record: record = {u16 d; stream}, lane half kg takes stream bits+// [(16 - 16 kg) K, (32 - 16 kg) K) plus the 15 - K bits after them (circular), so state(u), u = 0..15, is the 15 bits at+// (15 - u) K of the window.+template <int K>+__device__ __forceinline__ void stream_window(const uint8_t* rec, int kg, uint64_t& lo, uint64_t& hi) {+ const uint16_t* st = (const uint16_t*) (rec + 2);+ constexpr int NW = (15 * K + 15 + 15) / 16; // u16 words that hold 15 K + 15 bits+ uint64_t w[2] = {0ull, 0ull};+ if (kg == 1) {+#pragma unroll+ for (int i = 0; i < NW; ++i) w[i >> 2] |= (uint64_t) st[i] << (16 * (i & 3));+ } else { // the stream's upper half [16 K, 32 K) = K words; the bits past its end wrap to the stream's start+ const uint16_t* up = st + K;+#pragma unroll+ for (int i = 0; i < K; ++i) w[i >> 2] |= (uint64_t) up[i] << (16 * (i & 3));+ const uint64_t wrap = (uint64_t) st[0] & ((1ull << (15 - K)) - 1ull); // 15 - K <= 11 bits: inside word 0+ const int p = 16 * K;+ w[p >> 6] |= wrap << (p & 63); // (p & 63) + 15 - K <= 56: never past the word+ }+ lo = w[0];+ hi = w[1];+}+__device__ __forceinline__ uint32_t state_at(uint64_t lo, uint64_t hi, int pos) { // 15 bits at bit pos of the 128-bit window+ uint64_t v;+ if (pos + 15 <= 64) v = lo >> pos;+ else if (pos >= 64) v = hi >> (pos - 64);+ else v = (lo >> pos) | (hi << (64 - pos));+ return (uint32_t) v & 0x7fffu;+}++// ---------------------------------------------------------------- activations: where a lane's 8 values of an entry come from+// Each loader: valid(m) says entry m (this pass's column) exists; load8(m, k0, v) the 8 values k0..k0+7 as F32.+struct ActQ8 { // int8 rows with an F32 scale a 32: entry m = row rows[m] of q (n values a row), scales d (n / 32 a row)+ const int8_t* q;+ const float* d;+ const int* rows;+ int n;+ __device__ __forceinline__ void load8(int m, int k0, float* v) const {+ const size_t r = (size_t) rows[m];+ const int2 w = *(const int2*) (q + r * n + k0);+ const float sc = d[r * (n / 32) + (k0 >> 5)];+ const int8_t* b = (const int8_t*) &w;+#pragma unroll+ for (int j = 0; j < 8; ++j) v[j] = (float) b[j] * sc;+ }+};+struct ActF32 { // F32 rows: entry m = row rows[m] (ld floats a row)+ const float* x;+ const int* rows;+ size_t ld;+ __device__ __forceinline__ void load8(int m, int k0, float* v) const {+ const float4* p = (const float4*) (x + (size_t) rows[m] * ld + k0);+ const float4 a = p[0], b = p[1];+ v[0] = a.x; v[1] = a.y; v[2] = a.z; v[3] = a.w; v[4] = b.x; v[5] = b.y; v[6] = b.z; v[7] = b.w;+ }+};+struct ActF16 { // F16 rows: entry m = row rows[m] (ld halves a row)+ const uint16_t* x;+ const int* rows;+ size_t ld;+ __device__ __forceinline__ void load8(int m, int k0, float* v) const {+ const uint4 w = *(const uint4*) (x + (size_t) rows[m] * ld + k0);+ const uint32_t u[4] = {w.x, w.y, w.z, w.w};+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ v[2 * j] = half_bits_f(u[j] & 0xffffu);+ v[2 * j + 1] = half_bits_f(u[j] >> 16);+ }+ }+};++// ---------------------------------------------------------------- one 16-row tile over up to 16 entries+// acc[i] (i < 8) = D[entry 8 kg + i][row r] of the tile: sum over the row's k values of x_entry * w, k % 128 == 0.+// wrow: the tile's row r (this lane's) in the blob; nblk blocks.+template <int K, bool COMPUTED, class Act>+__device__ __forceinline__ void tile_dot(const uint8_t* __restrict__ wrow, int nblk, const uint32_t* lut, const Act& act, int ne,+ float* acc_out) {+ constexpr int bsz = 2 + 4 * K;+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;+ float tot[8];+#pragma unroll+ for (int i = 0; i < 8; ++i) tot[i] = 0.0f;+ const bool have = r < ne; // this lane's A rows (the entry m = r)+ for (int b = 0; b < nblk; ++b) {+ const uint8_t* rec = wrow + (size_t) b * bsz;+ const float d = half_bits_f(*(const uint16_t*) rec);+ uint64_t lo, hi;+ stream_window<K>(rec, kg, lo, hi);+#if defined(DS4_TQ_WMMA)+ float8_t acc = {0, 0, 0, 0, 0, 0, 0, 0};+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ half8_t wf, ah, al;+#pragma unroll+ for (int s2 = 0; s2 < 2; ++s2) {+ const int u = 2 * m + s2;+ uint32_t w0, w1;+ step_words<COMPUTED>(lut, state_at(lo, hi, (15 - u) * K), w0, w1);+ wf[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));+ wf[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));+ wf[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));+ wf[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));+ }+ float v[8];+ if (have) act.load8(r, b * 128 + 64 * kg + 8 * m, v);+ else {+#pragma unroll+ for (int j = 0; j < 8; ++j) v[j] = 0.0f;+ }+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const _Float16 h = (_Float16) v[j];+ ah[j] = h;+ al[j] = (_Float16) (v[j] - (float) h);+ }+ acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(ah, wf, acc); // D[m = entry][n = weight row]+ acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(al, wf, acc);+ }+#pragma unroll+ for (int i = 0; i < 8; ++i) tot[i] += d * acc[i];+#else+ // plain FMAs: this lane's row r, its 64 weights of the block against every entry (lane r + 16 kg' sums the halves)+ float part[kMaxEnt];+#pragma unroll+ for (int e = 0; e < kMaxEnt; ++e) part[e] = 0.0f;+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ float w[8];+#pragma unroll+ for (int s2 = 0; s2 < 2; ++s2) {+ uint32_t w0, w1;+ step_words<COMPUTED>(lut, state_at(lo, hi, (15 - (2 * m + s2)) * K), w0, w1);+ w[4 * s2 + 0] = half_bits_f(w0 & 0xffffu);+ w[4 * s2 + 1] = half_bits_f(w0 >> 16);+ w[4 * s2 + 2] = half_bits_f(w1 & 0xffffu);+ w[4 * s2 + 3] = half_bits_f(w1 >> 16);+ }+#pragma unroll+ for (int e = 0; e < kMaxEnt; ++e)+ if (e < ne) {+ float v[8];+ act.load8(e, b * 128 + 64 * kg + 8 * m, v);+#pragma unroll+ for (int j = 0; j < 8; ++j) part[e] = fmaf(w[j], v[j], part[e]);+ }+ }+ // entries 0..7 into tot[i] of the kg = 0 lane, 8..15 of the kg = 1 lane, so the output indexing matches the WMMA's+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const float p0 = part[i], p1 = part[8 + i];+ // lane kg holds its half's partial of both; the full sum needs the partner's half (same row, other kg)+ const float q0 = __shfl_xor(p0, 16), q1 = __shfl_xor(p1, 16);+ tot[i] += d * (kg == 0 ? p0 + q0 : p1 + q1);+ }+#endif+ }+#pragma unroll+ for (int i = 0; i < 8; ++i) acc_out[i] = tot[i];+}+// (the plain form: the block scale d is the row's, which both halves of the row read alike, so d * (sum of the halves) is+// right; the halves' partial sums swap through the shuffle, kg 0 keeping entries 0..7 and kg 1 entries 8..15)++__device__ __forceinline__ float swiglu(float gv, float uv, float limit) { // ds4_iq.cu's+ if (limit > 0.0f) {+ gv = fminf(gv, limit);+ uv = fminf(fmaxf(uv, -limit), limit);+ }+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);+}++template <int K, bool COMPUTED>+__device__ __forceinline__ void load_lut(uint32_t* lut) {+ if (!COMPUTED) {+ // the table as the CPU decoder has it (tq_cpu.cpp's int16 state table): fp16 * 2^13 rounded half away from zero, over+ // 2^13 - exact for values >= 2^-3 and within 2^-14 below; the result is still an exact fp16 (fewer significant bits)+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) {+ uint32_t pair = 0;+#pragma unroll+ for (int j = 0; j < 2; ++j) {+ const float v = half_bits_f(tq_lut_f16[2 * i + j]) * 8192.0f;+ const int n = (int) (v < 0.0f ? v - 0.5f : v + 0.5f);+ pair |= (uint32_t) __half_as_ushort(__float2half_rn((float) n / 8192.0f)) << (16 * j);+ }+ lut[i] = pair;+ }+ __syncthreads();+ }+}++// ---------------------------------------------------------------- token / window experts+__device__ __forceinline__ bool group_of(const Args& a, int grp, const uint8_t*& blob, int& m0, int& m1) {+ if (a.grp_ptr == nullptr) {+ const unsigned long long p = a.plan_ptr[grp];+ if (p == 0ull) return false;+ blob = (const uint8_t*) p;+ m0 = grp;+ m1 = grp + 1;+ return true;+ }+ if (grp >= *a.n_groups) return false;+ blob = (const uint8_t*) a.grp_ptr[grp];+ m0 = a.grp_start[grp];+ m1 = a.grp_start[grp + 1];+ return true;+}++// grid (n_ff / 64, groups): a wave = a 16-row tile of the gate and of the up of the group's expert, 16 entries a pass+template <int K, bool COMPUTED>+__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];+ __shared__ int ent[kMaxEnt], xr[kMaxEnt];+ const uint8_t* blob;+ int m0, m1;+ if (!group_of(a, blockIdx.y, blob, m0, m1)) return; // (uniform)+ load_lut<K, COMPUTED>(lut);+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;+ const int tile = blockIdx.x * kWaves + wave;+ const int row = tile * kTileRows + r;+ const size_t rb = (size_t) (a.n_embd / 128) * (2 + 4 * K);+ const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;+ const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += kMaxEnt) {+ const int ne = min(kMaxEnt, m1 - mm);+ __syncthreads();+ if ((int) threadIdx.x < ne) {+ const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;+ ent[threadIdx.x] = e;+ xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;+ }+ __syncthreads();+ if (tile * kTileRows >= a.n_ff) continue;+ const ActQ8 act{a.xq, a.xd, xr, a.n_embd};+ float ag[8], au[8];+ tile_dot<K, COMPUTED>(g, a.n_embd / 128, lut, act, ne, ag);+ tile_dot<K, COMPUTED>(u, a.n_embd / 128, lut, act, ne, au);+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const int m = 8 * kg + i;+ if (m < ne) a.h[(size_t) ent[m] * a.n_ff + row] = swiglu(ag[i], au[i], a.limit);+ }+ }+}++// grid (n_embd / 64, groups): the down's tile of a wave against the entries' int8 h+template <int K, bool COMPUTED>+__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];+ __shared__ int ent[kMaxEnt];+ const uint8_t* blob;+ int m0, m1;+ if (!group_of(a, blockIdx.y, blob, m0, m1)) return;+ load_lut<K, COMPUTED>(lut);+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;+ const int tile = blockIdx.x * kWaves + wave;+ const int row = tile * kTileRows + r;+ const size_t rb = (size_t) (a.n_ff / 128) * (2 + 4 * K);+ const uint8_t* d = blob + a.tr[2] + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += kMaxEnt) {+ const int ne = min(kMaxEnt, m1 - mm);+ __syncthreads();+ if ((int) threadIdx.x < ne) ent[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;+ __syncthreads();+ if (tile * kTileRows >= a.n_embd) continue;+ const ActQ8 act{a.hq, a.hd, ent, a.n_ff};+ float ad[8];+ tile_dot<K, COMPUTED>(d, a.n_ff / 128, lut, act, ne, ad);+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const int m = 8 * kg + i;+ if (m < ne) a.part[(size_t) ent[m] * a.n_embd + row] = ad[i];+ }+ }+}++// ---------------------------------------------------------------- the prompt path's sets+// grid (n_ff / 64, chunks of 16 rows, n_exp): expert e = blockIdx.z owns the sorted rows [bounds[e], bounds[e + 1]); local+// row = sorted row - row0. gate / up: F32 x[idx[local]] (ldx) -> F16 swiglu h[local * ldo + c]; down: F16 xh[local] (ldx) ->+// F32 y[local * ldo + c].+template <int K, bool COMPUTED, bool GATE_UP>+__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];+ __shared__ int rowsel[kMaxEnt];+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.y * kMaxEnt;+ if (r0 >= hi) return; // (uniform)+ load_lut<K, COMPUTED>(lut);+ const int ne = min(kMaxEnt, hi - r0);+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;+ const int N = GATE_UP ? a.n_ff : a.n_embd, Kin = GATE_UP ? a.n_embd : a.n_ff;+ const int tile = blockIdx.x * kWaves + wave;+ const int row = tile * kTileRows + r;+ const size_t rb = (size_t) (Kin / 128) * (2 + 4 * K);+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ if ((int) threadIdx.x < ne) rowsel[threadIdx.x] = GATE_UP ? a.idx[r0 + threadIdx.x] : r0 + (int) threadIdx.x;+ __syncthreads();+ if (tile * kTileRows >= N) return;+ float acc[8], accu[8];+ if (GATE_UP) {+ const ActF32 act{a.xf, rowsel, (size_t) a.ldx};+ tile_dot<K, COMPUTED>(blob + a.tr[0] + (size_t) row * rb, Kin / 128, lut, act, ne, acc);+ tile_dot<K, COMPUTED>(blob + a.tr[1] + (size_t) row * rb, Kin / 128, lut, act, ne, accu);+ } else {+ const ActF16 act{a.xh, rowsel, (size_t) a.ldx};+ tile_dot<K, COMPUTED>(blob + a.tr[2] + (size_t) row * rb, Kin / 128, lut, act, ne, acc);+ }+ const int kgl = threadIdx.x >> 4 & 1;+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const int m = 8 * kgl + i;+ if (m >= ne) continue;+ const size_t lr = (size_t) (r0 + m);+ if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(acc[i], accu[i], a.limit)));+ else a.y[lr * a.ldo + row] = acc[i];+ }+}++// ---------------------------------------------------------------- GEMV jobs+// grid (n / 64): y[c * ycs + r] = alpha (row r . x_c) + bias[r] for ncol (<= 16) int8 activation columns+template <int K, bool COMPUTED>+__global__ void __launch_bounds__(kThreads) mv_kernel(const MvJob j) {+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];+ __shared__ int cols[kMaxEnt];+ load_lut<K, COMPUTED>(lut);+ if ((int) threadIdx.x < kMaxEnt) cols[threadIdx.x] = (int) threadIdx.x;+ __syncthreads();+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;+ const int tile = blockIdx.x * kWaves + wave;+ if (tile * kTileRows >= j.n) return;+ const int row = tile * kTileRows + r;+ const ActQ8 act{j.xq, j.xd, cols, j.k};+ float acc[8];+ tile_dot<K, COMPUTED>(j.w + (size_t) row * j.ld, j.k / 128, lut, act, j.ncol, acc);+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const int m = 8 * kg + i;+ if (m < j.ncol && row < j.n) j.y[(size_t) m * j.ycs + row] = j.alpha * acc[i] + (j.bias != nullptr ? j.bias[row] : 0.0f);+ }+}++// ---------------------------------------------------------------- dispatch+#define DS4_TQ_SWITCH(t, F) \+ switch (t) { \+ case kTQK6: F(6, false); break; \+ case kTQK7: F(7, false); break; \+ case kTQK5X: F(5, true); break; \+ case kTQK4X: F(4, true); break; \+ case kTQK5T: F(5, false); break; \+ case kTQK4T: F(4, false); break; \+ default: return false; \+ }++bool bad(const char* what) {+ std::fprintf(stderr, "ds4 tq: %s\n", what);+ return false;+}+} // namespace++bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {+ (void) ne_max;+ if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("gate_up: geometry not covered");+ const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) groups);+#define GU_(K, C) gate_up_kernel<K, C><<<grid, kThreads, 0, s>>>(a)+ DS4_TQ_SWITCH(a.tg, GU_)+#undef GU_+ return true;+}++bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {+ (void) ne_max;+ if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("down: geometry not covered");+ const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) groups);+#define DN_(K, C) down_kernel<K, C><<<grid, kThreads, 0, s>>>(a)+ DS4_TQ_SWITCH(a.td, DN_)+#undef DN_+ return true;+}++bool set_gate_up(const SetArgs& a, cudaStream_t s) {+ if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("set_gate_up: geometry not covered");+ const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);+#define SG_(K, C) set_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a)+ DS4_TQ_SWITCH(a.tg, SG_)+#undef SG_+ return true;+}++bool set_down(const SetArgs& a, cudaStream_t s) {+ if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("set_down: geometry not covered");+ const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);+#define SD_(K, C) set_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a)+ DS4_TQ_SWITCH(a.td, SD_)+#undef SD_+ return true;+}++bool mv(const MvJob* jobs, int n, cudaStream_t s) {+ for (int i = 0; i < n; ++i) {+ const MvJob& j = jobs[i];+ if (j.k % 128 != 0 || j.n % kTileRows != 0 || j.ncol < 1 || j.ncol > kMaxEnt) return bad("mv: geometry not covered");+ const dim3 grid((unsigned) ((j.n + kWaves * kTileRows - 1) / (kWaves * kTileRows)));+#define MV_(K, C) mv_kernel<K, C><<<grid, kThreads, 0, s>>>(j)+ DS4_TQ_SWITCH(j.type, MV_)+#undef MV_+ }+ return true;+}++} // namespace strata::ds4::gguf::iq::tqdiff --git a/src/dsv41/ds4_tq_cpu.cpp b/src/dsv41/ds4_tq_cpu.cppnew file mode 100644index 000000000..afafb584a--- /dev/null+++ b/src/dsv41/ds4_tq_cpu.cpp@@ -0,0 +1,136 @@+// Strata. The trellis routed experts (ds4_iq.cuh's is_tq types) on the CPU lane of the DeepSeek-V4.1 engine path: Strata's+// own decode-and-dot kernels (include/strata/kernels/cpu/tq_cpu.hpp: AVX-512 / AVX2 / scalar, one result whatever the ISA)+// over the file's rows read in place. The activations are the device's (ds4_gguf.cu's q8_quant: int8 with an F32 scale a 32+// values), so an expert gives the same sums on either lane up to the order of the blocks' float sums.+#include "ds4_gguf_cpu.hpp"++#include <algorithm>+#include <cmath>+#include <cstdint>+#include <cstring>+#include <string>+#include <vector>++#if defined(STRATA_DSV41_TQ_CPU)+#include "strata/kernels/cpu/tq_cpu.hpp"+#endif++namespace strata::ds4::gguf {++#if defined(STRATA_DSV41_TQ_CPU)+namespace kc = strata::kernels::cpu;+namespace {++int kbits_of(int type) {+ switch (type) {+ case 145: return 6;+ case 146: return 7;+ case 224: case 226: return 5;+ case 225: case 227: return 4;+ default: return 0;+ }+}+constexpr int kActTq = 100; // IqCpu::gu_act / d_act of these types: the layout below++// an activation of k values: k / 32 F32 scales, then k int8+size_t act_bytes_of(int k) { return (size_t) k / 32 * 4 + (size_t) k; }++void unpack(const void* act, int k, kc::tq_detail::TqAct& a) {+ const uint8_t* p = (const uint8_t*) act;+ std::memcpy(a.d, p, (size_t) k / 32 * 4);+ const int8_t* q = (const int8_t*) (p + (size_t) k / 32 * 4);+ for (int i = 0; i < k; ++i) a.q[i] = q[i];+}++// dots of rows [r0, r1) of one matrix against nt activations: out[t][r]+void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {+ static thread_local std::vector<kc::tq_detail::TqAct> unp(kc::kTqMaxTokens);+ const kc::tq_detail::TqAct* ap[kc::kTqMaxTokens];+ for (int t = 0; t < nt; ++t) {+ unpack(act[t], n, unp[(size_t) t]);+ ap[t] = &unp[(size_t) t];+ }+ const int kb = kc::tq_kbits(type);+ const int16_t* tab = kc::tq_state_table_for(type);+ switch (kc::tq_isa()) {+ case kc::TqIsa::avx512: kc::tq_detail::tq_rows_avx512(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;+ case kc::TqIsa::avx2: kc::tq_detail::tq_rows_avx2(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;+ default: kc::tq_detail::tq_rows_scalar(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;+ }+}+} // namespace++bool tq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {+ if (kbits_of(tg) == 0 || kbits_of(td) == 0) {+ err = "not trellis types";+ return false;+ }+ if (n_embd % 128 != 0 || n_ff % 128 != 0 || n_embd > kc::kTqMaxN || n_ff > kc::kTqMaxN) {+ err = "the trellis CPU kernels take widths of 128 blocks up to " + std::to_string(kc::kTqMaxN);+ return false;+ }+ f.tg = tg;+ f.td = td;+ f.n_embd = n_embd;+ f.n_ff = n_ff;+ f.gu_act = f.d_act = kActTq;+ f.gu_row = (size_t) (n_embd / 128) * (2 + 4 * kbits_of(tg));+ f.d_row = (size_t) (n_ff / 128) * (2 + 4 * kbits_of(td));+ f.act_bytes = act_bytes_of(n_embd);+ f.h_bytes = act_bytes_of(n_ff);+ return true;+}++// x (k values) -> the layout above (the device's q8_quant: sc = max|x| / 127, q = round-half-even(x / sc))+void tq_cpu_quant(const float* x, int k, void* dst) {+ float* d = (float*) dst;+ int8_t* q = (int8_t*) ((uint8_t*) dst + (size_t) k / 32 * 4);+ for (int b = 0; b < k / 32; ++b) {+ const float* xb = x + b * 32;+ float am = 0.0f;+ for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(xb[j]));+ const float sc = am / 127.0f;+ d[b] = sc;+ for (int j = 0; j < 32; ++j) q[b * 32 + j] = (int8_t) (am > 0.0f ? (int) std::nearbyint(xb[j] / sc) : 0);+ }+}++void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,+ int r0, int r1, float limit) {+ static thread_local std::vector<float> tmp[kc::kTqMaxTokens];+ float* up_out[kc::kTqMaxTokens];+ for (int t = 0; t < nt; ++t) {+ tmp[t].resize((size_t) r1);+ up_out[t] = tmp[t].data();+ }+ dots(f.tg, gate, f.gu_row, f.n_embd, act, nt, ff, r0, r1);+ dots(f.tg, up, f.gu_row, f.n_embd, act, nt, up_out, r0, r1);+ for (int t = 0; t < nt; ++t)+ for (int r = r0; r < r1; ++r) {+ float g = ff[t][r], u = up_out[t][r];+ if (limit > 0.0f) {+ g = std::min(g, limit);+ u = std::min(std::max(u, -limit), limit);+ }+ ff[t][r] = g / (1.0f + std::exp(-g)) * u;+ }+}++void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {+ dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);+}+bool tq_cpu_type(int type) { return kbits_of(type) > 0; }+int tq_cpu_act() { return kActTq; }+#else+bool tq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {+ err = "this build has no CPU trellis kernels (STRATA_NATIVE_EXPERTS=OFF)";+ return false;+}+void tq_cpu_quant(const float*, int, void*) {}+void tq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}+void tq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}+bool tq_cpu_type(int) { return false; }+int tq_cpu_act() { return 100; }+#endif++} // namespace strata::ds4::ggufdiff --git a/tests/dsv41/tq_expert_test.cpp b/tests/dsv41/tq_expert_test.cppnew file mode 100644index 000000000..aac351178--- /dev/null+++ b/tests/dsv41/tq_expert_test.cpp@@ -0,0 +1,254 @@+// Strata. The trellis routed experts of the DeepSeek-V4.1 path (ds4_tq.cu on the device, ds4_tq_cpu.cpp on the CPU lane) against+// a double reference on host-decoded weights (tools/trellis/tqenc: decode_blocks), for the six types (TQK6 / TQK7 / TQK5X /+// TQK4X / TQK5T / TQK4T): the token path, the window path (grouped entries), the prompt sets (gate/up and down) and the CPU+// lane, plus the two lanes against each other. Random legal streams: any bit pattern is one.+// tq_expert_test [--type 145,146,...]+#include "../../src/dsv41/ds4_gguf_cpu.hpp"+#include "../../src/dsv41/ds4_iq.cuh"+#include "../../tools/trellis/tqenc.hpp"++#include <hip/hip_runtime.h>++#include <algorithm>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <vector>++namespace iq = strata::ds4::gguf::iq;+namespace gg = strata::ds4::gguf;+#define CK(x) do { hipError_t e_ = (x); if (e_ != hipSuccess) { std::printf("HIP error %s at %d\n", hipGetErrorString(e_), __LINE__); std::exit(2); } } while (0)++template <typename T> static T* up(const std::vector<T>& v) {+ T* d; CK(hipMalloc(&d, std::max<size_t>(1, v.size() * sizeof(T)))); CK(hipMemcpy(d, v.data(), v.size() * sizeof(T), hipMemcpyHostToDevice)); return d;+}+template <typename T> static std::vector<T> down_(const T* d, size_t n) {+ std::vector<T> v(n); CK(hipMemcpy(v.data(), d, n * sizeof(T), hipMemcpyDeviceToHost)); return v;+}+static float h2f(uint16_t h) {+ const int s = h >> 15, e = (h >> 10) & 31, m = h & 1023;+ float f = e == 0 ? std::ldexp((float) m, -24) : std::ldexp(1.0f + m / 1024.0f, e - 15);+ return s ? -f : f;+}+static uint16_t f2h(float f) { // round to nearest even, normal range+ uint32_t u; std::memcpy(&u, &f, 4);+ const uint32_t s = (u >> 16) & 0x8000u; int e = (int) ((u >> 23) & 0xff) - 127 + 15; uint32_t m = u & 0x7fffffu;+ uint32_t hm = m >> 13; const uint32_t rem = m & 0x1fffu;+ uint32_t v = ((uint32_t) e << 10) | hm;+ if (rem > 0x1000u || (rem == 0x1000u && (v & 1))) ++v;+ return (uint16_t) (s | v);+}++struct Q8 { std::vector<int8_t> q; std::vector<float> d; std::vector<float> xhat; };+// the engine's q8_quant: sc = max|x| / 127 per 32, q = round-half-even(x / sc)+static Q8 quant(const float* x, int k) {+ Q8 r; r.q.resize(k); r.d.resize(k / 32); r.xhat.resize(k);+ for (int b = 0; b < k / 32; ++b) {+ float am = 0; for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(x[b * 32 + j]));+ const float sc = am / 127.0f; r.d[b] = sc;+ for (int j = 0; j < 32; ++j) { const int q = am > 0 ? (int) std::nearbyint(x[b * 32 + j] / sc) : 0; r.q[b * 32 + j] = (int8_t) q; r.xhat[b * 32 + j] = (float) q * sc; }+ }+ return r;+}+static double swiglu(double g, double u, double lim) {+ if (lim > 0) { g = std::min(g, lim); u = std::min(std::max(u, -lim), lim); }+ return g / (1.0 + std::exp(-g)) * u;+}++struct Mat { std::vector<uint8_t> bytes; std::vector<float> w; int rows, k; };+static Mat make_mat(std::mt19937_64& rng, int type, int K, int rows, int k) {+ Mat m; m.rows = rows; m.k = k;+ const int bsz = 2 + 4 * K, nblk = k / 128;+ m.bytes.resize((size_t) rows * nblk * bsz);+ for (auto& b : m.bytes) b = (uint8_t) rng();+ for (int64_t b = 0; b < (int64_t) rows * nblk; ++b) { const uint16_t d = f2h(0.05f + 0.0005f * (float) (rng() % 600)); std::memcpy(&m.bytes[(size_t) b * bsz], &d, 2); }+ m.w.resize((size_t) rows * k);+ tqenc::decode_blocks(type, m.bytes.data(), (int64_t) rows * nblk, m.w.data());+ return m;+}+static double dotd(const float* w, const float* x, int k, double* sabs = nullptr) {+ double s = 0, sa = 0; for (int i = 0; i < k; ++i) { const double t = (double) w[i] * x[i]; s += t; sa += std::fabs(t); } if (sabs) *sabs = sa; return s;+}++int main(int argc, char** argv) {+ std::vector<int> types = {145, 146, 224, 225, 226, 227};+ for (int i = 1; i < argc; ++i)+ if (std::string(argv[i]) == "--type" && i + 1 < argc) { types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }+ std::mt19937_64 rng(11);+ std::normal_distribution<float> nd(0.f, 1.f);+ int fails = 0;+ auto report = [&](const char* what, int type, double worst, double tol) {+ const bool ok = worst <= tol;+ std::printf("%-6s %-22s worst %.2e (tol %.0e) %s\n", iq::name(type), what, worst, tol, ok ? "ok" : "FAIL");+ fails += !ok;+ };+ for (int type : types) {+ const int K = iq::tq_kbits(type);+ const int E = 512, FF = 256, NEXP = 3, T = 4;+ const float lim = 7.0f;+ const size_t rb_g = (size_t) (E / 128) * (2 + 4 * K), rb_d = (size_t) (FF / 128) * (2 + 4 * K);+ // experts: blob = gate | up | down, each 256-aligned+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);+ std::vector<Mat> G(NEXP), U(NEXP), D(NEXP);+ // h of row r of expert e for the input xv (F32 values), and the size of the operands it was made of+ auto refh = [&](int e, int r, const float* xv, double& scale) {+ double sg, su;+ const double g = dotd(&G[e].w[(size_t) r * E], xv, E, &sg), u = dotd(&U[e].w[(size_t) r * E], xv, E, &su);+ scale = sg + su + 1e-30;+ return swiglu(g, u, lim);+ };+ std::vector<uint8_t> blobs((size_t) NEXP * blob, 0);+ for (int e = 0; e < NEXP; ++e) {+ G[e] = make_mat(rng, type, K, FF, E); U[e] = make_mat(rng, type, K, FF, E); D[e] = make_mat(rng, type, K, E, FF);+ std::memcpy(&blobs[e * blob + off[0]], G[e].bytes.data(), G[e].bytes.size());+ std::memcpy(&blobs[e * blob + off[1]], U[e].bytes.data(), U[e].bytes.size());+ std::memcpy(&blobs[e * blob + off[2]], D[e].bytes.data(), D[e].bytes.size());+ }+ uint8_t* dblob = up(blobs);+ // tokens+ std::vector<float> x((size_t) T * E);+ for (auto& v : x) v = nd(rng) * (1.0f + 3.0f * (rng() % 7 == 0));+ std::vector<Q8> qx(T);+ std::vector<int8_t> xq((size_t) T * E); std::vector<float> xd((size_t) T * E / 32);+ for (int t = 0; t < T; ++t) { qx[t] = quant(&x[(size_t) t * E], E); std::memcpy(&xq[(size_t) t * E], qx[t].q.data(), E); std::memcpy(&xd[(size_t) t * E / 32], qx[t].d.data(), E / 32 * 4); }+ const int8_t* dxq = up(xq); const float* dxd = up(xd);++ // ---- token path: plan = 3 experts, input row 0+ {+ std::vector<unsigned long long> plan(NEXP);+ for (int e = 0; e < NEXP; ++e) plan[e] = (unsigned long long) (dblob + e * blob);+ iq::Args a{};+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = lim; a.xq = dxq; a.xd = dxd; a.tg = a.td = type;+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];+ float* dh; CK(hipMalloc(&dh, (size_t) NEXP * FF * 4)); a.h = dh;+ if (!iq::gate_up(a, NEXP, 1, 0)) { std::printf("gate_up refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const auto h = down_(dh, (size_t) NEXP * FF);+ double worst = 0;+ std::vector<float> href((size_t) NEXP * FF);+ for (int e = 0; e < NEXP; ++e)+ for (int r = 0; r < FF; ++r) {+ double sc;+ const double ref = refh(e, r, qx[0].xhat.data(), sc);+ href[(size_t) e * FF + r] = (float) ref;+ worst = std::max(worst, std::fabs(h[(size_t) e * FF + r] - ref) / sc);+ }+ report("token gate_up", type, worst, 1e-5);+ // down over the reference h, quantized+ std::vector<int8_t> hq((size_t) NEXP * FF); std::vector<float> hd((size_t) NEXP * FF / 32);+ std::vector<Q8> qh(NEXP);+ for (int e = 0; e < NEXP; ++e) { qh[e] = quant(&href[(size_t) e * FF], FF); std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }+ a.hq = up(hq); a.hd = up(hd);+ float* dp; CK(hipMalloc(&dp, (size_t) NEXP * E * 4)); a.part = dp;+ if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const auto part = down_(dp, (size_t) NEXP * E);+ worst = 0;+ for (int e = 0; e < NEXP; ++e)+ for (int r = 0; r < E; ++r) {+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], qh[e].xhat.data(), FF, &sa);+ worst = std::max(worst, std::fabs(part[(size_t) e * E + r] - ref) / (sa + 1e-30));+ }+ report("token down", type, worst, 1e-6);++ // ---- the CPU lane on the same experts and inputs; against the reference and the device+ gg::IqCpu fm; std::string err;+ if (!gg::iq_cpu_fmt(type, type, E, FF, fm, err)) { std::printf("cpu fmt: %s\n", err.c_str()); ++fails; }+ else {+ std::vector<uint8_t> act(fm.act_bytes), hact(fm.h_bytes);+ gg::iq_cpu_quant(fm.gu_act, x.data(), E, act.data());+ bool same = std::memcmp(act.data(), qx[0].d.data(), E / 32 * 4) == 0 && std::memcmp(act.data() + E / 32 * 4, qx[0].q.data(), E) == 0;+ std::printf("%-6s cpu activation == device quantizer: %s\n", iq::name(type), same ? "ok" : "FAIL");+ fails += !same;+ double wg = 0, wd = 0, wx = 0;+ for (int e = 0; e < NEXP; ++e) {+ std::vector<float> hc(FF), yc(E);+ float* hp[1] = {hc.data()}; float* yp[1] = {yc.data()};+ const void* ap[1] = {act.data()};+ gg::iq_cpu_gate_up(fm, &blobs[e * blob + off[0]], &blobs[e * blob + off[1]], ap, 1, hp, 0, FF, lim);+ for (int r = 0; r < FF; ++r) { double sc; refh(e, r, qx[0].xhat.data(), sc); wg = std::max(wg, std::fabs(hc[r] - href[(size_t) e * FF + r]) / sc); wx = std::max(wx, std::fabs(hc[r] - h[(size_t) e * FF + r]) / sc); }+ gg::iq_cpu_quant(fm.d_act, &href[(size_t) e * FF], FF, hact.data());+ const void* hpp[1] = {hact.data()};+ gg::iq_cpu_down(fm, &blobs[e * blob + off[2]], hpp, 1, yp, 0, E);+ for (int r = 0; r < E; ++r) { double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], qh[e].xhat.data(), FF, &sa); wd = std::max(wd, std::fabs(yc[r] - ref) / (sa + 1e-30)); }+ }+ report("cpu gate_up vs ref", type, wg, 1e-5);+ report("cpu down vs ref", type, wd, 1e-6);+ report("cpu vs device gate_up", type, wx, 1e-5);+ }+ }++ // ---- window path: groups of entries (entry e = t * 8 + i reads input row e >> 3)+ {+ std::vector<unsigned long long> grp = {(unsigned long long) (dblob + 0 * blob), (unsigned long long) (dblob + 2 * blob)};+ std::vector<int> start = {0, 3, 5}, ent = {1, 10, 19, 2, 8}, ng = {2};+ iq::Args a{};+ a.grp_ptr = up(grp); a.grp_start = up(start); a.grp_ent = up(ent); a.n_groups = up(ng);+ a.n_embd = E; a.n_ff = FF; a.limit = lim; a.xq = dxq; a.xd = dxd; a.tg = a.td = type;+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];+ const int NE = 24;+ float* dh; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMemset(dh, 0, (size_t) NE * FF * 4)); a.h = dh;+ iq::gate_up(a, 2, 3, 0);+ CK(hipDeviceSynchronize());+ const auto h = down_(dh, (size_t) NE * FF);+ double worst = 0;+ for (int gi = 0; gi < 2; ++gi)+ for (int j = start[gi]; j < start[gi + 1]; ++j) {+ const int en = ent[j], t = en >> 3, e = gi == 0 ? 0 : 2;+ for (int r = 0; r < FF; ++r) {+ double sc;+ const double ref = refh(e, r, qx[t].xhat.data(), sc);+ worst = std::max(worst, std::fabs(h[(size_t) en * FF + r] - ref) / sc);+ }+ }+ report("window gate_up", type, worst, 1e-5);+ }++ // ---- the prompt sets: expert 0 owns sorted rows [0, 37), expert 2 rows [37, 42); input rows idx[sorted]+ {+ const int NR = 42;+ std::vector<int> bounds = {0, 37, 37, 42}, idx(NR);+ for (int i = 0; i < NR; ++i) idx[i] = i % T;+ iq::SetArgs q{};+ q.wbase = dblob; q.stride = blob; for (int r = 0; r < 3; ++r) q.tr[r] = off[r];+ q.bounds = up(bounds); q.row0 = 0; q.max_rows = 37; q.n_exp = 3; q.n_embd = E; q.n_ff = FF; q.limit = lim;+ q.tg = q.td = type; q.xf = up(x); q.idx = up(idx); q.ldx = E;+ uint16_t* dh16; CK(hipMalloc(&dh16, (size_t) NR * FF * 2)); q.h = dh16; q.ldo = FF;+ iq::set_gate_up(q, 0);+ CK(hipDeviceSynchronize());+ const auto h16 = down_(dh16, (size_t) NR * FF);+ double worst = 0;+ for (int e = 0; e < 3; ++e)+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr)+ for (int r = 0; r < FF; ++r) {+ double sc;+ const double ref = refh(e, r, &x[(size_t) idx[lr] * E], sc);+ worst = std::max(worst, std::fabs(h2f(h16[(size_t) lr * FF + r]) - ref) / sc);+ }+ report("set gate_up (f16 out)", type, worst, 1e-3);+ // down over those f16 h rows+ q.xh = dh16; q.ldx = FF;+ float* dy; CK(hipMalloc(&dy, (size_t) NR * E * 4)); q.y = dy; q.ldo = E;+ iq::set_down(q, 0);+ CK(hipDeviceSynchronize());+ const auto y = down_(dy, (size_t) NR * E);+ worst = 0;+ for (int e = 0; e < 3; ++e)+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {+ std::vector<float> hf(FF);+ for (int c = 0; c < FF; ++c) hf[c] = h2f(h16[(size_t) lr * FF + c]);+ for (int r = 0; r < E; ++r) {+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hf.data(), FF, &sa);+ worst = std::max(worst, std::fabs(y[(size_t) lr * E + r] - ref) / (sa + 1e-30));+ }+ }+ report("set down", type, worst, 1e-6);+ }+ }+ std::printf("%s\n", fails ? "FAILED" : "ALL OK");+ return fails;+}