Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: trellis routed experts end to end on the device and the CPU lane (TQK6/7, TQK5X/4X computed, TQK5T/4T table): ds4_tq.cu (gfx12 WMMA + portable FMA tiles over the file layout), ds4_tq_cpu.cpp (Strata tq_cpu kernels), iq entry points route the types; test ds4_tq_test (all six types pass: device and CPU lane within 1e-6 of a double reference)

Niko1221e5b4021c6f0d
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5ea05b40b..e3478900b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -776,12 +776,30 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_include_directories(strata_dsv41_kernels PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})
add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cu
- src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp)
+ src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
+ src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)
target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})
find_package(Threads REQUIRED)
target_link_libraries(strata-dsv41 PRIVATE Threads::Threads)
+ if(STRATA_NATIVE_EXPERTS) # Strata's CPU trellis kernels (tq_cpu.hpp) for the trellis experts on the CPU lane
+ target_link_libraries(strata-dsv41 PRIVATE strata_kernels_cpu)
+ target_compile_definitions(strata-dsv41 PRIVATE STRATA_DSV41_TQ_CPU=1)
+ endif()
+ if(STRATA_NATIVE_EXPERTS)
+ # the trellis experts (device kernels ds4_tq.cu, CPU lane ds4_tq_cpu.cpp) against a double reference; needs a GPU, no model
+ set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE HIP)
+ add_executable(ds4_tq_test tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp
+ src/dsv41/ds4_iq_cpu.cpp tools/trellis/tqenc.cpp)
+ target_include_directories(ds4_tq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
+ ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)
+ target_link_libraries(ds4_tq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)
+ target_compile_definitions(ds4_tq_test PRIVATE STRATA_DSV41_TQ_CPU=1)
+ if(STRATA_ENABLE_HIP)
+ target_compile_options(ds4_tq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
+ endif()
+ endif()
if(STRATA_ENABLE_HIP)
foreach(_t strata_dsv41_kernels strata-dsv41)
target_compile_options(${_t} PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hpp
index ac6d6f176..7555fccfb 100644
--- a/include/strata/kernels/cpu/tq_cpu.hpp
+++ b/include/strata/kernels/cpu/tq_cpu.hpp
@@ -74,7 +74,7 @@ void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, si
/// Largest token group per call and widest row the kernels take.
inline constexpr int kTqMaxTokens = 8;
-inline constexpr int kTqMaxN = 4096;
+inline constexpr int kTqMaxN = 5120; // (DeepSeek-V4.1 hidden width; was 4096)
namespace tq_detail {
/// One token's Q8_0 activation unpacked for the kernels: the int8 values widened to int16 and the scales as floats.
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 556d8c1bc..9b8e87014 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1002,9 +1002,9 @@ int launch_errors() { return g_errors.load(std::memory_order_relaxed); }
size_t type_row_bytes(int type, int64_t k) {
if (type == kTypeMXFP4) return k % kMxBlock == 0 ? mx_row_bytes(k) : 0;
- return iq::supported(type) && k % 256 == 0 ? iq::row_bytes(type, k) : 0;
+ return iq::supported(type) && k % iq::block_values(type) == 0 ? iq::row_bytes(type, k) : 0;
}
-int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? 256 : 0; }
+int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? iq::block_values(type) : 0; }
const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }
namespace {
@@ -1024,8 +1024,8 @@ bool iq_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit, iq:
ia.limit = limit;
ia.tg = expert_type(lay, 0);
ia.td = expert_type(lay, 2);
- if (ia.tg != expert_type(lay, 1) || !iq::supported(ia.tg) || !iq::supported(ia.td) || n_embd % 256 != 0 ||
- n_ff % 256 != 0) {
+ if (ia.tg != expert_type(lay, 1) || !iq::supported(ia.tg) || !iq::supported(ia.td) || n_embd % iq::block_values(ia.tg) != 0 ||
+ n_ff % iq::block_values(ia.td) != 0) {
std::fprintf(stderr, "ds4 gguf %s: expert types %s / %s / %s (n_embd %d, n_ff %d) not covered\n", who,
type_name(expert_type(lay, 0)), type_name(expert_type(lay, 1)), type_name(expert_type(lay, 2)),
n_embd, n_ff);
diff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hpp
index 32b40c70a..286afd2ea 100644
--- a/src/dsv41/ds4_gguf_cpu.hpp
+++ b/src/dsv41/ds4_gguf_cpu.hpp
@@ -36,6 +36,16 @@ void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, cons
float* const* ff, int r0, int r1, float limit);
/// out[t][r] = down_r . hq[t] for rows [r0, r1).
void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+/// The trellis types (ds4_iq.cuh's is_tq; ds4_tq_cpu.cpp, Strata's tq_cpu kernels): the same four calls, reached from the
+/// iq_cpu_* ones above for those types (their activations: F32 scale a 32 values, then the int8 values).
+bool tq_cpu_type(int type);
+int tq_cpu_act();
+bool tq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);
+void tq_cpu_quant(const float* x, int k, void* dst);
+void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+ int r0, int r1, float limit);
+void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+
/// The CPU lane's MXFP4 rows in the device's arithmetic (ds4_gguf.cu's mx_dot): the activations rounded to int8 per 32
/// values as the device rounds them (q8_round's q and scale), each block's products summed exactly in integers, then
/// times (weight scale x activation scale) into F32 sums - the same numbers as the device up to the order of the
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index ec2c70819..ba62fccce 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -910,7 +910,10 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {
bool supported(int t) { return block_bytes(t) > 0; }
+int block_values(int t) { return is_tq(t) ? 128 : block_bytes(t) > 0 ? 256 : 0; }
+
int block_bytes(int t) {
+ if (is_tq(t)) return 2 + 4 * tq_kbits(t);
switch (t) {
case kIQ2_XXS: return Ty<kIQ2_XXS>::B;
case kIQ2_S: return Ty<kIQ2_S>::B;
@@ -926,6 +929,12 @@ int block_bytes(int t) {
const char* name(int t) {
switch (t) {
+ case kTQK6: return "TQK6";
+ case kTQK7: return "TQK7";
+ case kTQK5X: return "TQK5X";
+ case kTQK4X: return "TQK4X";
+ case kTQK5T: return "TQK5T";
+ case kTQK4T: return "TQK4T";
case kIQ2_XXS: return "IQ2_XXS";
case kIQ2_S: return "IQ2_S";
case kIQ3_XXS: return "IQ3_XXS";
@@ -939,6 +948,7 @@ const char* name(int t) {
}
bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ if (is_tq(a.tg)) return tq::gate_up(a, groups, ne_max, s);
if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;
const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);
#define GU_(T) gu_ne<T>(a, grid, ne_max, s)
@@ -948,6 +958,7 @@ bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
}
bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ if (is_tq(a.td)) return tq::down(a, groups, ne_max, s);
if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;
const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);
#define DN_(T) dn_ne<T>(a, grid, ne_max, s)
@@ -957,15 +968,18 @@ bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
}
bool light_gate_up(const SetArgs& a, cudaStream_t s) {
+ if (is_tq(a.tg)) return false; // (the trellis sets take the tiles for every expert)
if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;
return light_set(a, true, s);
}
bool light_down(const SetArgs& a, cudaStream_t s) {
+ if (is_tq(a.td)) return false;
if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;
return light_set(a, false, s);
}
bool set_gate_up(const SetArgs& a, cudaStream_t s) {
+ if (is_tq(a.tg)) return tq::set_gate_up(a, s);
if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;
SetArgs t = a;
if (light_gate_up(a, s)) { // the experts of a few rows; the tiles the others
@@ -980,6 +994,7 @@ bool set_gate_up(const SetArgs& a, cudaStream_t s) {
}
bool set_down(const SetArgs& a, cudaStream_t s) {
+ if (is_tq(a.td)) return tq::set_down(a, s);
if (a.n_ff % 256 != 0 || a.n_embd % BN != 0) return false;
SetArgs t = a;
if (light_down(a, s)) {
@@ -994,6 +1009,17 @@ bool set_down(const SetArgs& a, cudaStream_t s) {
}
bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+ { // the trellis jobs go to their kernels, the others on
+ MvJob tqj[64], oth[64];
+ int nt = 0, no = 0;
+ if (n > 64) return false;
+ for (int i = 0; i < n; ++i) (is_tq(jobs[i].type) ? tqj[nt++] : oth[no++]) = jobs[i];
+ if (nt > 0) {
+ if (!tq::mv(tqj, nt, s)) return false;
+ if (no == 0) return true;
+ return mv(oth, no, s);
+ }
+ }
// the jobs by type, up to kMaxJobs a launch
bool done[64] = {};
if (n > 64) return false;
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index 65711f517..f13199fdc 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -22,12 +22,23 @@ namespace strata::ds4::gguf::iq {
constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,
kIQ4_XS = 23;
+/// The trellis types (Strata's, ds4_tq.cu): ggml's TQK6 / TQK7 (145 / 146, the table codebook) and the experimental
+/// TQK5X / TQK4X (224 / 225, computed codebook) and TQK5T / TQK4T (226 / 227, table): a block of 128 values is
+/// {fp16 d; 32 steps of K bits}, 2 + 4 K bytes (tools/trellis/tqenc.hpp). They go through the same entry points below.
+constexpr int kTQK6 = 145, kTQK7 = 146, kTQK5X = 224, kTQK4X = 225, kTQK5T = 226, kTQK4T = 227;
+constexpr int tq_kbits(int type) {
+ return type == kTQK6 ? 6 : type == kTQK7 ? 7 : (type == kTQK5X || type == kTQK5T) ? 5 : (type == kTQK4X || type == kTQK4T) ? 4 : 0;
+}
+constexpr bool is_tq(int type) { return tq_kbits(type) > 0; }
+
/// Whether the device kernels take this ggml type.
bool supported(int type);
-/// Bytes of a block (256 values) of the type (0: not one of these).
+/// Values a block of the type holds (256; the trellis types 128; 0: not one of these).
+int block_values(int type);
+/// Bytes of a block of the type (0: not one of these).
int block_bytes(int type);
-/// Bytes of a row of k values (k % 256 == 0).
-inline size_t row_bytes(int type, int64_t k) { return (size_t) (k / 256) * (size_t) block_bytes(type); }
+/// Bytes of a row of k values (k % block_values == 0).
+inline size_t row_bytes(int type, int64_t k) { return (size_t) (k / block_values(type)) * (size_t) block_bytes(type); }
/// The type's name ("?" when unknown).
const char* name(int type);
@@ -101,4 +112,14 @@ struct MvJob {
};
bool mv(const MvJob* jobs, int n, cudaStream_t s);
+/// The trellis types' kernels (ds4_tq.cu), reached from the entry points above for those types: arithmetic as the CPU lane's
+/// (ds4_tq_cpu.cpp: int8 activations with an F32 scale a 32, exact weights), on the tensor cores of gfx12 (WMMA) or in FMAs.
+namespace tq {
+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s);
+bool down(const Args& a, int groups, int ne_max, cudaStream_t s);
+bool set_gate_up(const SetArgs& a, cudaStream_t s);
+bool set_down(const SetArgs& a, cudaStream_t s);
+bool mv(const MvJob* jobs, int n, cudaStream_t s);
+} // namespace tq
+
} // namespace strata::ds4::gguf::iq
diff --git a/src/dsv41/ds4_iq_cpu.cpp b/src/dsv41/ds4_iq_cpu.cpp
index f96c81952..e0f04678a 100644
--- a/src/dsv41/ds4_iq_cpu.cpp
+++ b/src/dsv41/ds4_iq_cpu.cpp
@@ -756,14 +756,22 @@ void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int
bool iq_cpu_own_rows(int type) { return own(type); }
#else
-bool iq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
- err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts' CPU lane is not in it";
+// (no ggml-cpu lane for the low-bit types here; the trellis types have their own: ds4_tq_cpu.cpp)
+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+ if (tq_cpu_type(tg) && tq_cpu_type(td)) return tq_cpu_fmt(tg, td, n_embd, n_ff, f, err);
+ err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts CPU lane is not in it";
return false;
}
-void iq_cpu_quant(int, const float*, int, void*) {}
-void iq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int,
- float) {}
-void iq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
+void iq_cpu_quant(int act, const float* x, int k, void* dst) {
+ if (act == tq_cpu_act()) tq_cpu_quant(x, k, dst);
+}
+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+ int r0, int r1, float limit) {
+ if (tq_cpu_type(f.tg)) tq_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);
+}
+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+ if (tq_cpu_type(f.td)) tq_cpu_down(f, down, hq, nt, out, r0, r1);
+}
bool iq_cpu_own_rows(int) { return false; }
#endif
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 369b51387..f30a982eb 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1478,7 +1478,7 @@ struct Model::Impl {
// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
static int gg_type_of(const std::string& dtype) {
- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23})
+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})
if (dtype == gg::type_name(t)) return t;
return -1;
}
diff --git a/src/dsv41/ds4_tq.cu b/src/dsv41/ds4_tq.cu
new file mode 100644
index 000000000..63883c646
--- /dev/null
+++ b/src/dsv41/ds4_tq.cu
@@ -0,0 +1,465 @@
+// Strata. The trellis routed experts (TQK6 / TQK7 / TQK5X / TQK4X / TQK5T / TQK4T, ds4_iq.cuh's is_tq) on the device for the
+// DeepSeek-V4.1 engine path. The weights stay as the file has them (a row of k values = k / 128 blocks of {fp16 d; 4 K stream
+// bytes}); the blob in VRAM is the file's bytes, so the expert cache, the tensor-parallel cuts and the CPU lane read the same
+// layout. Arithmetic as the CPU lane's (ds4_tq_cpu.cpp): the activations are int8 with an F32 scale per 32 values (the
+// engine's q8_quant), the weights exact (fp16 codebook values times the block's fp16 scale); the sums are F32.
+//
+// gfx12 (RDNA4, R9700): v_wmma_f32_16x16x16_f16. The weights are the B operand (16 rows of a tile = the 16 columns, a lane
+// holds one row's 8 consecutive k), the activations the A operand (16 entries = the 16 rows, split x = hi + lo in two fp16
+// so the product keeps ~2^-22), D[entry][weight row] in F32: a lane holds the entries 8 kg .. 8 kg + 7 of its row, so the
+// block scale d is the lane's own. Lane l = r + 16 kg owns row r and the 16 steps 16 kg .. 16 kg + 15 of every block, whose
+// 15-bit states are one 128-bit window of the stream (the tile code of tools/dsv41_lab/wmma_emu.hpp, checked there against a
+// double reference and on the card by tests/dsv41/tqw_hip_test.cpp). Elsewhere the same lane mapping in plain FMAs.
+#include "ds4_iq.cuh"
+
+#include <cuda_fp16.h>
+
+#include "ggml-common-tq.h" // tq_lut_f16 (the 2048-point codebook as fp16 bit patterns, __device__)
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+
+#if defined(STRATA_USE_HIP) && (defined(__GFX12__) || defined(__gfx1200__) || defined(__gfx1201__))
+#define DS4_TQ_WMMA 1
+#endif
+
+namespace strata::ds4::gguf::iq::tq {
+namespace {
+
+constexpr int kWaves = 4, kThreads = kWaves * 32; // a block: 4 waves, a wave = one 16-row tile of each matrix
+constexpr int kTileRows = 16;
+constexpr int kMaxEnt = 16; // entries (columns) a pass
+
+#if defined(DS4_TQ_WMMA)
+typedef _Float16 half8_t __attribute__((ext_vector_type(8)));
+typedef float float8_t __attribute__((ext_vector_type(8)));
+#endif
+
+constexpr bool computed_code(int type) { return type == kTQK5X || type == kTQK4X; }
+
+// ---------------------------------------------------------------- the codebook and the stream
+// a step's 15-bit state -> the four weights as two words of two fp16 (lo, hi): w0 w1 | w2 w3
+template <bool COMPUTED>
+__device__ __forceinline__ void step_words(const uint32_t* lut, uint32_t s, uint32_t& w0, uint32_t& w1) {
+ if (COMPUTED) {
+ w0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+ w1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+ } else {
+ const uint32_t x = s * 0x9e3779b1u;
+ w0 = lut[x >> 21];
+ w1 = lut[(x >> 10) & 2047u];
+ }
+}
+__device__ __forceinline__ float half_bits_f(uint32_t b) { return __half2float(__ushort_as_half((unsigned short) b)); }
+
+// the 128-bit window of a lane's 16 steps in one block record: record = {u16 d; stream}, lane half kg takes stream bits
+// [(16 - 16 kg) K, (32 - 16 kg) K) plus the 15 - K bits after them (circular), so state(u), u = 0..15, is the 15 bits at
+// (15 - u) K of the window.
+template <int K>
+__device__ __forceinline__ void stream_window(const uint8_t* rec, int kg, uint64_t& lo, uint64_t& hi) {
+ const uint16_t* st = (const uint16_t*) (rec + 2);
+ constexpr int NW = (15 * K + 15 + 15) / 16; // u16 words that hold 15 K + 15 bits
+ uint64_t w[2] = {0ull, 0ull};
+ if (kg == 1) {
+#pragma unroll
+ for (int i = 0; i < NW; ++i) w[i >> 2] |= (uint64_t) st[i] << (16 * (i & 3));
+ } else { // the stream's upper half [16 K, 32 K) = K words; the bits past its end wrap to the stream's start
+ const uint16_t* up = st + K;
+#pragma unroll
+ for (int i = 0; i < K; ++i) w[i >> 2] |= (uint64_t) up[i] << (16 * (i & 3));
+ const uint64_t wrap = (uint64_t) st[0] & ((1ull << (15 - K)) - 1ull); // 15 - K <= 11 bits: inside word 0
+ const int p = 16 * K;
+ w[p >> 6] |= wrap << (p & 63); // (p & 63) + 15 - K <= 56: never past the word
+ }
+ lo = w[0];
+ hi = w[1];
+}
+__device__ __forceinline__ uint32_t state_at(uint64_t lo, uint64_t hi, int pos) { // 15 bits at bit pos of the 128-bit window
+ uint64_t v;
+ if (pos + 15 <= 64) v = lo >> pos;
+ else if (pos >= 64) v = hi >> (pos - 64);
+ else v = (lo >> pos) | (hi << (64 - pos));
+ return (uint32_t) v & 0x7fffu;
+}
+
+// ---------------------------------------------------------------- activations: where a lane's 8 values of an entry come from
+// Each loader: valid(m) says entry m (this pass's column) exists; load8(m, k0, v) the 8 values k0..k0+7 as F32.
+struct ActQ8 { // int8 rows with an F32 scale a 32: entry m = row rows[m] of q (n values a row), scales d (n / 32 a row)
+ const int8_t* q;
+ const float* d;
+ const int* rows;
+ int n;
+ __device__ __forceinline__ void load8(int m, int k0, float* v) const {
+ const size_t r = (size_t) rows[m];
+ const int2 w = *(const int2*) (q + r * n + k0);
+ const float sc = d[r * (n / 32) + (k0 >> 5)];
+ const int8_t* b = (const int8_t*) &w;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) v[j] = (float) b[j] * sc;
+ }
+};
+struct ActF32 { // F32 rows: entry m = row rows[m] (ld floats a row)
+ const float* x;
+ const int* rows;
+ size_t ld;
+ __device__ __forceinline__ void load8(int m, int k0, float* v) const {
+ const float4* p = (const float4*) (x + (size_t) rows[m] * ld + k0);
+ const float4 a = p[0], b = p[1];
+ v[0] = a.x; v[1] = a.y; v[2] = a.z; v[3] = a.w; v[4] = b.x; v[5] = b.y; v[6] = b.z; v[7] = b.w;
+ }
+};
+struct ActF16 { // F16 rows: entry m = row rows[m] (ld halves a row)
+ const uint16_t* x;
+ const int* rows;
+ size_t ld;
+ __device__ __forceinline__ void load8(int m, int k0, float* v) const {
+ const uint4 w = *(const uint4*) (x + (size_t) rows[m] * ld + k0);
+ const uint32_t u[4] = {w.x, w.y, w.z, w.w};
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ v[2 * j] = half_bits_f(u[j] & 0xffffu);
+ v[2 * j + 1] = half_bits_f(u[j] >> 16);
+ }
+ }
+};
+
+// ---------------------------------------------------------------- one 16-row tile over up to 16 entries
+// acc[i] (i < 8) = D[entry 8 kg + i][row r] of the tile: sum over the row's k values of x_entry * w, k % 128 == 0.
+// wrow: the tile's row r (this lane's) in the blob; nblk blocks.
+template <int K, bool COMPUTED, class Act>
+__device__ __forceinline__ void tile_dot(const uint8_t* __restrict__ wrow, int nblk, const uint32_t* lut, const Act& act, int ne,
+ float* acc_out) {
+ constexpr int bsz = 2 + 4 * K;
+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;
+ float tot[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) tot[i] = 0.0f;
+ const bool have = r < ne; // this lane's A rows (the entry m = r)
+ for (int b = 0; b < nblk; ++b) {
+ const uint8_t* rec = wrow + (size_t) b * bsz;
+ const float d = half_bits_f(*(const uint16_t*) rec);
+ uint64_t lo, hi;
+ stream_window<K>(rec, kg, lo, hi);
+#if defined(DS4_TQ_WMMA)
+ float8_t acc = {0, 0, 0, 0, 0, 0, 0, 0};
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ half8_t wf, ah, al;
+#pragma unroll
+ for (int s2 = 0; s2 < 2; ++s2) {
+ const int u = 2 * m + s2;
+ uint32_t w0, w1;
+ step_words<COMPUTED>(lut, state_at(lo, hi, (15 - u) * K), w0, w1);
+ wf[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));
+ wf[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));
+ wf[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));
+ wf[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));
+ }
+ float v[8];
+ if (have) act.load8(r, b * 128 + 64 * kg + 8 * m, v);
+ else {
+#pragma unroll
+ for (int j = 0; j < 8; ++j) v[j] = 0.0f;
+ }
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const _Float16 h = (_Float16) v[j];
+ ah[j] = h;
+ al[j] = (_Float16) (v[j] - (float) h);
+ }
+ acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(ah, wf, acc); // D[m = entry][n = weight row]
+ acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(al, wf, acc);
+ }
+#pragma unroll
+ for (int i = 0; i < 8; ++i) tot[i] += d * acc[i];
+#else
+ // plain FMAs: this lane's row r, its 64 weights of the block against every entry (lane r + 16 kg' sums the halves)
+ float part[kMaxEnt];
+#pragma unroll
+ for (int e = 0; e < kMaxEnt; ++e) part[e] = 0.0f;
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ float w[8];
+#pragma unroll
+ for (int s2 = 0; s2 < 2; ++s2) {
+ uint32_t w0, w1;
+ step_words<COMPUTED>(lut, state_at(lo, hi, (15 - (2 * m + s2)) * K), w0, w1);
+ w[4 * s2 + 0] = half_bits_f(w0 & 0xffffu);
+ w[4 * s2 + 1] = half_bits_f(w0 >> 16);
+ w[4 * s2 + 2] = half_bits_f(w1 & 0xffffu);
+ w[4 * s2 + 3] = half_bits_f(w1 >> 16);
+ }
+#pragma unroll
+ for (int e = 0; e < kMaxEnt; ++e)
+ if (e < ne) {
+ float v[8];
+ act.load8(e, b * 128 + 64 * kg + 8 * m, v);
+#pragma unroll
+ for (int j = 0; j < 8; ++j) part[e] = fmaf(w[j], v[j], part[e]);
+ }
+ }
+ // entries 0..7 into tot[i] of the kg = 0 lane, 8..15 of the kg = 1 lane, so the output indexing matches the WMMA's
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const float p0 = part[i], p1 = part[8 + i];
+ // lane kg holds its half's partial of both; the full sum needs the partner's half (same row, other kg)
+ const float q0 = __shfl_xor(p0, 16), q1 = __shfl_xor(p1, 16);
+ tot[i] += d * (kg == 0 ? p0 + q0 : p1 + q1);
+ }
+#endif
+ }
+#pragma unroll
+ for (int i = 0; i < 8; ++i) acc_out[i] = tot[i];
+}
+// (the plain form: the block scale d is the row's, which both halves of the row read alike, so d * (sum of the halves) is
+// right; the halves' partial sums swap through the shuffle, kg 0 keeping entries 0..7 and kg 1 entries 8..15)
+
+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) { // ds4_iq.cu's
+ if (limit > 0.0f) {
+ gv = fminf(gv, limit);
+ uv = fminf(fmaxf(uv, -limit), limit);
+ }
+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
+}
+
+template <int K, bool COMPUTED>
+__device__ __forceinline__ void load_lut(uint32_t* lut) {
+ if (!COMPUTED) {
+ // the table as the CPU decoder has it (tq_cpu.cpp's int16 state table): fp16 * 2^13 rounded half away from zero, over
+ // 2^13 - exact for values >= 2^-3 and within 2^-14 below; the result is still an exact fp16 (fewer significant bits)
+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) {
+ uint32_t pair = 0;
+#pragma unroll
+ for (int j = 0; j < 2; ++j) {
+ const float v = half_bits_f(tq_lut_f16[2 * i + j]) * 8192.0f;
+ const int n = (int) (v < 0.0f ? v - 0.5f : v + 0.5f);
+ pair |= (uint32_t) __half_as_ushort(__float2half_rn((float) n / 8192.0f)) << (16 * j);
+ }
+ lut[i] = pair;
+ }
+ __syncthreads();
+ }
+}
+
+// ---------------------------------------------------------------- token / window experts
+__device__ __forceinline__ bool group_of(const Args& a, int grp, const uint8_t*& blob, int& m0, int& m1) {
+ if (a.grp_ptr == nullptr) {
+ const unsigned long long p = a.plan_ptr[grp];
+ if (p == 0ull) return false;
+ blob = (const uint8_t*) p;
+ m0 = grp;
+ m1 = grp + 1;
+ return true;
+ }
+ if (grp >= *a.n_groups) return false;
+ blob = (const uint8_t*) a.grp_ptr[grp];
+ m0 = a.grp_start[grp];
+ m1 = a.grp_start[grp + 1];
+ return true;
+}
+
+// grid (n_ff / 64, groups): a wave = a 16-row tile of the gate and of the up of the group's expert, 16 entries a pass
+template <int K, bool COMPUTED>
+__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+ __shared__ int ent[kMaxEnt], xr[kMaxEnt];
+ const uint8_t* blob;
+ int m0, m1;
+ if (!group_of(a, blockIdx.y, blob, m0, m1)) return; // (uniform)
+ load_lut<K, COMPUTED>(lut);
+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+ const int tile = blockIdx.x * kWaves + wave;
+ const int row = tile * kTileRows + r;
+ const size_t rb = (size_t) (a.n_embd / 128) * (2 + 4 * K);
+ const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
+ const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += kMaxEnt) {
+ const int ne = min(kMaxEnt, m1 - mm);
+ __syncthreads();
+ if ((int) threadIdx.x < ne) {
+ const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+ ent[threadIdx.x] = e;
+ xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;
+ }
+ __syncthreads();
+ if (tile * kTileRows >= a.n_ff) continue;
+ const ActQ8 act{a.xq, a.xd, xr, a.n_embd};
+ float ag[8], au[8];
+ tile_dot<K, COMPUTED>(g, a.n_embd / 128, lut, act, ne, ag);
+ tile_dot<K, COMPUTED>(u, a.n_embd / 128, lut, act, ne, au);
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const int m = 8 * kg + i;
+ if (m < ne) a.h[(size_t) ent[m] * a.n_ff + row] = swiglu(ag[i], au[i], a.limit);
+ }
+ }
+}
+
+// grid (n_embd / 64, groups): the down's tile of a wave against the entries' int8 h
+template <int K, bool COMPUTED>
+__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {
+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+ __shared__ int ent[kMaxEnt];
+ const uint8_t* blob;
+ int m0, m1;
+ if (!group_of(a, blockIdx.y, blob, m0, m1)) return;
+ load_lut<K, COMPUTED>(lut);
+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+ const int tile = blockIdx.x * kWaves + wave;
+ const int row = tile * kTileRows + r;
+ const size_t rb = (size_t) (a.n_ff / 128) * (2 + 4 * K);
+ const uint8_t* d = blob + a.tr[2] + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += kMaxEnt) {
+ const int ne = min(kMaxEnt, m1 - mm);
+ __syncthreads();
+ if ((int) threadIdx.x < ne) ent[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+ __syncthreads();
+ if (tile * kTileRows >= a.n_embd) continue;
+ const ActQ8 act{a.hq, a.hd, ent, a.n_ff};
+ float ad[8];
+ tile_dot<K, COMPUTED>(d, a.n_ff / 128, lut, act, ne, ad);
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const int m = 8 * kg + i;
+ if (m < ne) a.part[(size_t) ent[m] * a.n_embd + row] = ad[i];
+ }
+ }
+}
+
+// ---------------------------------------------------------------- the prompt path's sets
+// grid (n_ff / 64, chunks of 16 rows, n_exp): expert e = blockIdx.z owns the sorted rows [bounds[e], bounds[e + 1]); local
+// row = sorted row - row0. gate / up: F32 x[idx[local]] (ldx) -> F16 swiglu h[local * ldo + c]; down: F16 xh[local] (ldx) ->
+// F32 y[local * ldo + c].
+template <int K, bool COMPUTED, bool GATE_UP>
+__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {
+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+ __shared__ int rowsel[kMaxEnt];
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.y * kMaxEnt;
+ if (r0 >= hi) return; // (uniform)
+ load_lut<K, COMPUTED>(lut);
+ const int ne = min(kMaxEnt, hi - r0);
+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+ const int N = GATE_UP ? a.n_ff : a.n_embd, Kin = GATE_UP ? a.n_embd : a.n_ff;
+ const int tile = blockIdx.x * kWaves + wave;
+ const int row = tile * kTileRows + r;
+ const size_t rb = (size_t) (Kin / 128) * (2 + 4 * K);
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ if ((int) threadIdx.x < ne) rowsel[threadIdx.x] = GATE_UP ? a.idx[r0 + threadIdx.x] : r0 + (int) threadIdx.x;
+ __syncthreads();
+ if (tile * kTileRows >= N) return;
+ float acc[8], accu[8];
+ if (GATE_UP) {
+ const ActF32 act{a.xf, rowsel, (size_t) a.ldx};
+ tile_dot<K, COMPUTED>(blob + a.tr[0] + (size_t) row * rb, Kin / 128, lut, act, ne, acc);
+ tile_dot<K, COMPUTED>(blob + a.tr[1] + (size_t) row * rb, Kin / 128, lut, act, ne, accu);
+ } else {
+ const ActF16 act{a.xh, rowsel, (size_t) a.ldx};
+ tile_dot<K, COMPUTED>(blob + a.tr[2] + (size_t) row * rb, Kin / 128, lut, act, ne, acc);
+ }
+ const int kgl = threadIdx.x >> 4 & 1;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const int m = 8 * kgl + i;
+ if (m >= ne) continue;
+ const size_t lr = (size_t) (r0 + m);
+ if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(acc[i], accu[i], a.limit)));
+ else a.y[lr * a.ldo + row] = acc[i];
+ }
+}
+
+// ---------------------------------------------------------------- GEMV jobs
+// grid (n / 64): y[c * ycs + r] = alpha (row r . x_c) + bias[r] for ncol (<= 16) int8 activation columns
+template <int K, bool COMPUTED>
+__global__ void __launch_bounds__(kThreads) mv_kernel(const MvJob j) {
+ __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+ __shared__ int cols[kMaxEnt];
+ load_lut<K, COMPUTED>(lut);
+ if ((int) threadIdx.x < kMaxEnt) cols[threadIdx.x] = (int) threadIdx.x;
+ __syncthreads();
+ const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+ const int tile = blockIdx.x * kWaves + wave;
+ if (tile * kTileRows >= j.n) return;
+ const int row = tile * kTileRows + r;
+ const ActQ8 act{j.xq, j.xd, cols, j.k};
+ float acc[8];
+ tile_dot<K, COMPUTED>(j.w + (size_t) row * j.ld, j.k / 128, lut, act, j.ncol, acc);
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const int m = 8 * kg + i;
+ if (m < j.ncol && row < j.n) j.y[(size_t) m * j.ycs + row] = j.alpha * acc[i] + (j.bias != nullptr ? j.bias[row] : 0.0f);
+ }
+}
+
+// ---------------------------------------------------------------- dispatch
+#define DS4_TQ_SWITCH(t, F) \
+ switch (t) { \
+ case kTQK6: F(6, false); break; \
+ case kTQK7: F(7, false); break; \
+ case kTQK5X: F(5, true); break; \
+ case kTQK4X: F(4, true); break; \
+ case kTQK5T: F(5, false); break; \
+ case kTQK4T: F(4, false); break; \
+ default: return false; \
+ }
+
+bool bad(const char* what) {
+ std::fprintf(stderr, "ds4 tq: %s\n", what);
+ return false;
+}
+} // namespace
+
+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ (void) ne_max;
+ if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("gate_up: geometry not covered");
+ const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) groups);
+#define GU_(K, C) gate_up_kernel<K, C><<<grid, kThreads, 0, s>>>(a)
+ DS4_TQ_SWITCH(a.tg, GU_)
+#undef GU_
+ return true;
+}
+
+bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ (void) ne_max;
+ if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("down: geometry not covered");
+ const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) groups);
+#define DN_(K, C) down_kernel<K, C><<<grid, kThreads, 0, s>>>(a)
+ DS4_TQ_SWITCH(a.td, DN_)
+#undef DN_
+ return true;
+}
+
+bool set_gate_up(const SetArgs& a, cudaStream_t s) {
+ if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("set_gate_up: geometry not covered");
+ const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);
+#define SG_(K, C) set_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a)
+ DS4_TQ_SWITCH(a.tg, SG_)
+#undef SG_
+ return true;
+}
+
+bool set_down(const SetArgs& a, cudaStream_t s) {
+ if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("set_down: geometry not covered");
+ const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);
+#define SD_(K, C) set_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a)
+ DS4_TQ_SWITCH(a.td, SD_)
+#undef SD_
+ return true;
+}
+
+bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+ for (int i = 0; i < n; ++i) {
+ const MvJob& j = jobs[i];
+ if (j.k % 128 != 0 || j.n % kTileRows != 0 || j.ncol < 1 || j.ncol > kMaxEnt) return bad("mv: geometry not covered");
+ const dim3 grid((unsigned) ((j.n + kWaves * kTileRows - 1) / (kWaves * kTileRows)));
+#define MV_(K, C) mv_kernel<K, C><<<grid, kThreads, 0, s>>>(j)
+ DS4_TQ_SWITCH(j.type, MV_)
+#undef MV_
+ }
+ return true;
+}
+
+} // namespace strata::ds4::gguf::iq::tq
diff --git a/src/dsv41/ds4_tq_cpu.cpp b/src/dsv41/ds4_tq_cpu.cpp
new file mode 100644
index 000000000..afafb584a
--- /dev/null
+++ b/src/dsv41/ds4_tq_cpu.cpp
@@ -0,0 +1,136 @@
+// Strata. The trellis routed experts (ds4_iq.cuh's is_tq types) on the CPU lane of the DeepSeek-V4.1 engine path: Strata's
+// own decode-and-dot kernels (include/strata/kernels/cpu/tq_cpu.hpp: AVX-512 / AVX2 / scalar, one result whatever the ISA)
+// over the file's rows read in place. The activations are the device's (ds4_gguf.cu's q8_quant: int8 with an F32 scale a 32
+// values), so an expert gives the same sums on either lane up to the order of the blocks' float sums.
+#include "ds4_gguf_cpu.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#if defined(STRATA_DSV41_TQ_CPU)
+#include "strata/kernels/cpu/tq_cpu.hpp"
+#endif
+
+namespace strata::ds4::gguf {
+
+#if defined(STRATA_DSV41_TQ_CPU)
+namespace kc = strata::kernels::cpu;
+namespace {
+
+int kbits_of(int type) {
+ switch (type) {
+ case 145: return 6;
+ case 146: return 7;
+ case 224: case 226: return 5;
+ case 225: case 227: return 4;
+ default: return 0;
+ }
+}
+constexpr int kActTq = 100; // IqCpu::gu_act / d_act of these types: the layout below
+
+// an activation of k values: k / 32 F32 scales, then k int8
+size_t act_bytes_of(int k) { return (size_t) k / 32 * 4 + (size_t) k; }
+
+void unpack(const void* act, int k, kc::tq_detail::TqAct& a) {
+ const uint8_t* p = (const uint8_t*) act;
+ std::memcpy(a.d, p, (size_t) k / 32 * 4);
+ const int8_t* q = (const int8_t*) (p + (size_t) k / 32 * 4);
+ for (int i = 0; i < k; ++i) a.q[i] = q[i];
+}
+
+// dots of rows [r0, r1) of one matrix against nt activations: out[t][r]
+void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
+ static thread_local std::vector<kc::tq_detail::TqAct> unp(kc::kTqMaxTokens);
+ const kc::tq_detail::TqAct* ap[kc::kTqMaxTokens];
+ for (int t = 0; t < nt; ++t) {
+ unpack(act[t], n, unp[(size_t) t]);
+ ap[t] = &unp[(size_t) t];
+ }
+ const int kb = kc::tq_kbits(type);
+ const int16_t* tab = kc::tq_state_table_for(type);
+ switch (kc::tq_isa()) {
+ case kc::TqIsa::avx512: kc::tq_detail::tq_rows_avx512(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
+ case kc::TqIsa::avx2: kc::tq_detail::tq_rows_avx2(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
+ default: kc::tq_detail::tq_rows_scalar(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
+ }
+}
+} // namespace
+
+bool tq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+ if (kbits_of(tg) == 0 || kbits_of(td) == 0) {
+ err = "not trellis types";
+ return false;
+ }
+ if (n_embd % 128 != 0 || n_ff % 128 != 0 || n_embd > kc::kTqMaxN || n_ff > kc::kTqMaxN) {
+ err = "the trellis CPU kernels take widths of 128 blocks up to " + std::to_string(kc::kTqMaxN);
+ return false;
+ }
+ f.tg = tg;
+ f.td = td;
+ f.n_embd = n_embd;
+ f.n_ff = n_ff;
+ f.gu_act = f.d_act = kActTq;
+ f.gu_row = (size_t) (n_embd / 128) * (2 + 4 * kbits_of(tg));
+ f.d_row = (size_t) (n_ff / 128) * (2 + 4 * kbits_of(td));
+ f.act_bytes = act_bytes_of(n_embd);
+ f.h_bytes = act_bytes_of(n_ff);
+ return true;
+}
+
+// x (k values) -> the layout above (the device's q8_quant: sc = max|x| / 127, q = round-half-even(x / sc))
+void tq_cpu_quant(const float* x, int k, void* dst) {
+ float* d = (float*) dst;
+ int8_t* q = (int8_t*) ((uint8_t*) dst + (size_t) k / 32 * 4);
+ for (int b = 0; b < k / 32; ++b) {
+ const float* xb = x + b * 32;
+ float am = 0.0f;
+ for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(xb[j]));
+ const float sc = am / 127.0f;
+ d[b] = sc;
+ for (int j = 0; j < 32; ++j) q[b * 32 + j] = (int8_t) (am > 0.0f ? (int) std::nearbyint(xb[j] / sc) : 0);
+ }
+}
+
+void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+ int r0, int r1, float limit) {
+ static thread_local std::vector<float> tmp[kc::kTqMaxTokens];
+ float* up_out[kc::kTqMaxTokens];
+ for (int t = 0; t < nt; ++t) {
+ tmp[t].resize((size_t) r1);
+ up_out[t] = tmp[t].data();
+ }
+ dots(f.tg, gate, f.gu_row, f.n_embd, act, nt, ff, r0, r1);
+ dots(f.tg, up, f.gu_row, f.n_embd, act, nt, up_out, r0, r1);
+ for (int t = 0; t < nt; ++t)
+ for (int r = r0; r < r1; ++r) {
+ float g = ff[t][r], u = up_out[t][r];
+ if (limit > 0.0f) {
+ g = std::min(g, limit);
+ u = std::min(std::max(u, -limit), limit);
+ }
+ ff[t][r] = g / (1.0f + std::exp(-g)) * u;
+ }
+}
+
+void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+ dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);
+}
+bool tq_cpu_type(int type) { return kbits_of(type) > 0; }
+int tq_cpu_act() { return kActTq; }
+#else
+bool tq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
+ err = "this build has no CPU trellis kernels (STRATA_NATIVE_EXPERTS=OFF)";
+ return false;
+}
+void tq_cpu_quant(const float*, int, void*) {}
+void tq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}
+void tq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
+bool tq_cpu_type(int) { return false; }
+int tq_cpu_act() { return 100; }
+#endif
+
+} // namespace strata::ds4::gguf
diff --git a/tests/dsv41/tq_expert_test.cpp b/tests/dsv41/tq_expert_test.cpp
new file mode 100644
index 000000000..aac351178
--- /dev/null
+++ b/tests/dsv41/tq_expert_test.cpp
@@ -0,0 +1,254 @@
+// Strata. The trellis routed experts of the DeepSeek-V4.1 path (ds4_tq.cu on the device, ds4_tq_cpu.cpp on the CPU lane) against
+// a double reference on host-decoded weights (tools/trellis/tqenc: decode_blocks), for the six types (TQK6 / TQK7 / TQK5X /
+// TQK4X / TQK5T / TQK4T): the token path, the window path (grouped entries), the prompt sets (gate/up and down) and the CPU
+// lane, plus the two lanes against each other. Random legal streams: any bit pattern is one.
+// tq_expert_test [--type 145,146,...]
+#include "../../src/dsv41/ds4_gguf_cpu.hpp"
+#include "../../src/dsv41/ds4_iq.cuh"
+#include "../../tools/trellis/tqenc.hpp"
+
+#include <hip/hip_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace iq = strata::ds4::gguf::iq;
+namespace gg = strata::ds4::gguf;
+#define CK(x) do { hipError_t e_ = (x); if (e_ != hipSuccess) { std::printf("HIP error %s at %d\n", hipGetErrorString(e_), __LINE__); std::exit(2); } } while (0)
+
+template <typename T> static T* up(const std::vector<T>& v) {
+ T* d; CK(hipMalloc(&d, std::max<size_t>(1, v.size() * sizeof(T)))); CK(hipMemcpy(d, v.data(), v.size() * sizeof(T), hipMemcpyHostToDevice)); return d;
+}
+template <typename T> static std::vector<T> down_(const T* d, size_t n) {
+ std::vector<T> v(n); CK(hipMemcpy(v.data(), d, n * sizeof(T), hipMemcpyDeviceToHost)); return v;
+}
+static float h2f(uint16_t h) {
+ const int s = h >> 15, e = (h >> 10) & 31, m = h & 1023;
+ float f = e == 0 ? std::ldexp((float) m, -24) : std::ldexp(1.0f + m / 1024.0f, e - 15);
+ return s ? -f : f;
+}
+static uint16_t f2h(float f) { // round to nearest even, normal range
+ uint32_t u; std::memcpy(&u, &f, 4);
+ const uint32_t s = (u >> 16) & 0x8000u; int e = (int) ((u >> 23) & 0xff) - 127 + 15; uint32_t m = u & 0x7fffffu;
+ uint32_t hm = m >> 13; const uint32_t rem = m & 0x1fffu;
+ uint32_t v = ((uint32_t) e << 10) | hm;
+ if (rem > 0x1000u || (rem == 0x1000u && (v & 1))) ++v;
+ return (uint16_t) (s | v);
+}
+
+struct Q8 { std::vector<int8_t> q; std::vector<float> d; std::vector<float> xhat; };
+// the engine's q8_quant: sc = max|x| / 127 per 32, q = round-half-even(x / sc)
+static Q8 quant(const float* x, int k) {
+ Q8 r; r.q.resize(k); r.d.resize(k / 32); r.xhat.resize(k);
+ for (int b = 0; b < k / 32; ++b) {
+ float am = 0; for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(x[b * 32 + j]));
+ const float sc = am / 127.0f; r.d[b] = sc;
+ for (int j = 0; j < 32; ++j) { const int q = am > 0 ? (int) std::nearbyint(x[b * 32 + j] / sc) : 0; r.q[b * 32 + j] = (int8_t) q; r.xhat[b * 32 + j] = (float) q * sc; }
+ }
+ return r;
+}
+static double swiglu(double g, double u, double lim) {
+ if (lim > 0) { g = std::min(g, lim); u = std::min(std::max(u, -lim), lim); }
+ return g / (1.0 + std::exp(-g)) * u;
+}
+
+struct Mat { std::vector<uint8_t> bytes; std::vector<float> w; int rows, k; };
+static Mat make_mat(std::mt19937_64& rng, int type, int K, int rows, int k) {
+ Mat m; m.rows = rows; m.k = k;
+ const int bsz = 2 + 4 * K, nblk = k / 128;
+ m.bytes.resize((size_t) rows * nblk * bsz);
+ for (auto& b : m.bytes) b = (uint8_t) rng();
+ for (int64_t b = 0; b < (int64_t) rows * nblk; ++b) { const uint16_t d = f2h(0.05f + 0.0005f * (float) (rng() % 600)); std::memcpy(&m.bytes[(size_t) b * bsz], &d, 2); }
+ m.w.resize((size_t) rows * k);
+ tqenc::decode_blocks(type, m.bytes.data(), (int64_t) rows * nblk, m.w.data());
+ return m;
+}
+static double dotd(const float* w, const float* x, int k, double* sabs = nullptr) {
+ double s = 0, sa = 0; for (int i = 0; i < k; ++i) { const double t = (double) w[i] * x[i]; s += t; sa += std::fabs(t); } if (sabs) *sabs = sa; return s;
+}
+
+int main(int argc, char** argv) {
+ std::vector<int> types = {145, 146, 224, 225, 226, 227};
+ for (int i = 1; i < argc; ++i)
+ if (std::string(argv[i]) == "--type" && i + 1 < argc) { types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
+ std::mt19937_64 rng(11);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ int fails = 0;
+ auto report = [&](const char* what, int type, double worst, double tol) {
+ const bool ok = worst <= tol;
+ std::printf("%-6s %-22s worst %.2e (tol %.0e) %s\n", iq::name(type), what, worst, tol, ok ? "ok" : "FAIL");
+ fails += !ok;
+ };
+ for (int type : types) {
+ const int K = iq::tq_kbits(type);
+ const int E = 512, FF = 256, NEXP = 3, T = 4;
+ const float lim = 7.0f;
+ const size_t rb_g = (size_t) (E / 128) * (2 + 4 * K), rb_d = (size_t) (FF / 128) * (2 + 4 * K);
+ // experts: blob = gate | up | down, each 256-aligned
+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };
+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
+ std::vector<Mat> G(NEXP), U(NEXP), D(NEXP);
+ // h of row r of expert e for the input xv (F32 values), and the size of the operands it was made of
+ auto refh = [&](int e, int r, const float* xv, double& scale) {
+ double sg, su;
+ const double g = dotd(&G[e].w[(size_t) r * E], xv, E, &sg), u = dotd(&U[e].w[(size_t) r * E], xv, E, &su);
+ scale = sg + su + 1e-30;
+ return swiglu(g, u, lim);
+ };
+ std::vector<uint8_t> blobs((size_t) NEXP * blob, 0);
+ for (int e = 0; e < NEXP; ++e) {
+ G[e] = make_mat(rng, type, K, FF, E); U[e] = make_mat(rng, type, K, FF, E); D[e] = make_mat(rng, type, K, E, FF);
+ std::memcpy(&blobs[e * blob + off[0]], G[e].bytes.data(), G[e].bytes.size());
+ std::memcpy(&blobs[e * blob + off[1]], U[e].bytes.data(), U[e].bytes.size());
+ std::memcpy(&blobs[e * blob + off[2]], D[e].bytes.data(), D[e].bytes.size());
+ }
+ uint8_t* dblob = up(blobs);
+ // tokens
+ std::vector<float> x((size_t) T * E);
+ for (auto& v : x) v = nd(rng) * (1.0f + 3.0f * (rng() % 7 == 0));
+ std::vector<Q8> qx(T);
+ std::vector<int8_t> xq((size_t) T * E); std::vector<float> xd((size_t) T * E / 32);
+ for (int t = 0; t < T; ++t) { qx[t] = quant(&x[(size_t) t * E], E); std::memcpy(&xq[(size_t) t * E], qx[t].q.data(), E); std::memcpy(&xd[(size_t) t * E / 32], qx[t].d.data(), E / 32 * 4); }
+ const int8_t* dxq = up(xq); const float* dxd = up(xd);
+
+ // ---- token path: plan = 3 experts, input row 0
+ {
+ std::vector<unsigned long long> plan(NEXP);
+ for (int e = 0; e < NEXP; ++e) plan[e] = (unsigned long long) (dblob + e * blob);
+ iq::Args a{};
+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = lim; a.xq = dxq; a.xd = dxd; a.tg = a.td = type;
+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
+ float* dh; CK(hipMalloc(&dh, (size_t) NEXP * FF * 4)); a.h = dh;
+ if (!iq::gate_up(a, NEXP, 1, 0)) { std::printf("gate_up refused\n"); return 1; }
+ CK(hipDeviceSynchronize());
+ const auto h = down_(dh, (size_t) NEXP * FF);
+ double worst = 0;
+ std::vector<float> href((size_t) NEXP * FF);
+ for (int e = 0; e < NEXP; ++e)
+ for (int r = 0; r < FF; ++r) {
+ double sc;
+ const double ref = refh(e, r, qx[0].xhat.data(), sc);
+ href[(size_t) e * FF + r] = (float) ref;
+ worst = std::max(worst, std::fabs(h[(size_t) e * FF + r] - ref) / sc);
+ }
+ report("token gate_up", type, worst, 1e-5);
+ // down over the reference h, quantized
+ std::vector<int8_t> hq((size_t) NEXP * FF); std::vector<float> hd((size_t) NEXP * FF / 32);
+ std::vector<Q8> qh(NEXP);
+ for (int e = 0; e < NEXP; ++e) { qh[e] = quant(&href[(size_t) e * FF], FF); std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }
+ a.hq = up(hq); a.hd = up(hd);
+ float* dp; CK(hipMalloc(&dp, (size_t) NEXP * E * 4)); a.part = dp;
+ if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }
+ CK(hipDeviceSynchronize());
+ const auto part = down_(dp, (size_t) NEXP * E);
+ worst = 0;
+ for (int e = 0; e < NEXP; ++e)
+ for (int r = 0; r < E; ++r) {
+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], qh[e].xhat.data(), FF, &sa);
+ worst = std::max(worst, std::fabs(part[(size_t) e * E + r] - ref) / (sa + 1e-30));
+ }
+ report("token down", type, worst, 1e-6);
+
+ // ---- the CPU lane on the same experts and inputs; against the reference and the device
+ gg::IqCpu fm; std::string err;
+ if (!gg::iq_cpu_fmt(type, type, E, FF, fm, err)) { std::printf("cpu fmt: %s\n", err.c_str()); ++fails; }
+ else {
+ std::vector<uint8_t> act(fm.act_bytes), hact(fm.h_bytes);
+ gg::iq_cpu_quant(fm.gu_act, x.data(), E, act.data());
+ bool same = std::memcmp(act.data(), qx[0].d.data(), E / 32 * 4) == 0 && std::memcmp(act.data() + E / 32 * 4, qx[0].q.data(), E) == 0;
+ std::printf("%-6s cpu activation == device quantizer: %s\n", iq::name(type), same ? "ok" : "FAIL");
+ fails += !same;
+ double wg = 0, wd = 0, wx = 0;
+ for (int e = 0; e < NEXP; ++e) {
+ std::vector<float> hc(FF), yc(E);
+ float* hp[1] = {hc.data()}; float* yp[1] = {yc.data()};
+ const void* ap[1] = {act.data()};
+ gg::iq_cpu_gate_up(fm, &blobs[e * blob + off[0]], &blobs[e * blob + off[1]], ap, 1, hp, 0, FF, lim);
+ for (int r = 0; r < FF; ++r) { double sc; refh(e, r, qx[0].xhat.data(), sc); wg = std::max(wg, std::fabs(hc[r] - href[(size_t) e * FF + r]) / sc); wx = std::max(wx, std::fabs(hc[r] - h[(size_t) e * FF + r]) / sc); }
+ gg::iq_cpu_quant(fm.d_act, &href[(size_t) e * FF], FF, hact.data());
+ const void* hpp[1] = {hact.data()};
+ gg::iq_cpu_down(fm, &blobs[e * blob + off[2]], hpp, 1, yp, 0, E);
+ for (int r = 0; r < E; ++r) { double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], qh[e].xhat.data(), FF, &sa); wd = std::max(wd, std::fabs(yc[r] - ref) / (sa + 1e-30)); }
+ }
+ report("cpu gate_up vs ref", type, wg, 1e-5);
+ report("cpu down vs ref", type, wd, 1e-6);
+ report("cpu vs device gate_up", type, wx, 1e-5);
+ }
+ }
+
+ // ---- window path: groups of entries (entry e = t * 8 + i reads input row e >> 3)
+ {
+ std::vector<unsigned long long> grp = {(unsigned long long) (dblob + 0 * blob), (unsigned long long) (dblob + 2 * blob)};
+ std::vector<int> start = {0, 3, 5}, ent = {1, 10, 19, 2, 8}, ng = {2};
+ iq::Args a{};
+ a.grp_ptr = up(grp); a.grp_start = up(start); a.grp_ent = up(ent); a.n_groups = up(ng);
+ a.n_embd = E; a.n_ff = FF; a.limit = lim; a.xq = dxq; a.xd = dxd; a.tg = a.td = type;
+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
+ const int NE = 24;
+ float* dh; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMemset(dh, 0, (size_t) NE * FF * 4)); a.h = dh;
+ iq::gate_up(a, 2, 3, 0);
+ CK(hipDeviceSynchronize());
+ const auto h = down_(dh, (size_t) NE * FF);
+ double worst = 0;
+ for (int gi = 0; gi < 2; ++gi)
+ for (int j = start[gi]; j < start[gi + 1]; ++j) {
+ const int en = ent[j], t = en >> 3, e = gi == 0 ? 0 : 2;
+ for (int r = 0; r < FF; ++r) {
+ double sc;
+ const double ref = refh(e, r, qx[t].xhat.data(), sc);
+ worst = std::max(worst, std::fabs(h[(size_t) en * FF + r] - ref) / sc);
+ }
+ }
+ report("window gate_up", type, worst, 1e-5);
+ }
+
+ // ---- the prompt sets: expert 0 owns sorted rows [0, 37), expert 2 rows [37, 42); input rows idx[sorted]
+ {
+ const int NR = 42;
+ std::vector<int> bounds = {0, 37, 37, 42}, idx(NR);
+ for (int i = 0; i < NR; ++i) idx[i] = i % T;
+ iq::SetArgs q{};
+ q.wbase = dblob; q.stride = blob; for (int r = 0; r < 3; ++r) q.tr[r] = off[r];
+ q.bounds = up(bounds); q.row0 = 0; q.max_rows = 37; q.n_exp = 3; q.n_embd = E; q.n_ff = FF; q.limit = lim;
+ q.tg = q.td = type; q.xf = up(x); q.idx = up(idx); q.ldx = E;
+ uint16_t* dh16; CK(hipMalloc(&dh16, (size_t) NR * FF * 2)); q.h = dh16; q.ldo = FF;
+ iq::set_gate_up(q, 0);
+ CK(hipDeviceSynchronize());
+ const auto h16 = down_(dh16, (size_t) NR * FF);
+ double worst = 0;
+ for (int e = 0; e < 3; ++e)
+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr)
+ for (int r = 0; r < FF; ++r) {
+ double sc;
+ const double ref = refh(e, r, &x[(size_t) idx[lr] * E], sc);
+ worst = std::max(worst, std::fabs(h2f(h16[(size_t) lr * FF + r]) - ref) / sc);
+ }
+ report("set gate_up (f16 out)", type, worst, 1e-3);
+ // down over those f16 h rows
+ q.xh = dh16; q.ldx = FF;
+ float* dy; CK(hipMalloc(&dy, (size_t) NR * E * 4)); q.y = dy; q.ldo = E;
+ iq::set_down(q, 0);
+ CK(hipDeviceSynchronize());
+ const auto y = down_(dy, (size_t) NR * E);
+ worst = 0;
+ for (int e = 0; e < 3; ++e)
+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {
+ std::vector<float> hf(FF);
+ for (int c = 0; c < FF; ++c) hf[c] = h2f(h16[(size_t) lr * FF + c]);
+ for (int r = 0; r < E; ++r) {
+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hf.data(), FF, &sa);
+ worst = std::max(worst, std::fabs(y[(size_t) lr * E + r] - ref) / (sa + 1e-30));
+ }
+ }
+ report("set down", type, worst, 1e-6);
+ }
+ }
+ std::printf("%s\n", fails ? "FAILED" : "ALL OK");
+ return fails;
+}