Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: IQ2_XS device kernels (bit-exact vs ggml), Q5_K/Q4_K/Q6_K embedding rows on the host, i-quant expert test, CPU-lane bench reference
Niko122149c157fae318
diff --git a/CMakeLists.txt b/CMakeLists.txtindex bcf1734e4..ff5409244 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1,4 +1,4 @@-# Strata Engine - top-level build (P1.S1).+# Strata Engine - top-level build (P1.S1).## Targets (phase-1 §P1.S1): `strata_artifact` (no CUDA dependency), `tests`. `strata_plan` and the C++# `strata-pack` are added when their sources exist - a target with no sources would make the build graph@@ -799,14 +799,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)# the trellis experts (device kernels ds4_tq.cu, CPU lane ds4_tq_cpu.cpp) against a double reference; needs a GPU, no modelset_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE HIP)add_executable(ds4_tq_test tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp- src/dsv41/ds4_iq_cpu.cpp tools/trellis/tqenc.cpp)+ src/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)target_include_directories(ds4_tq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)target_link_libraries(ds4_tq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)target_compile_definitions(ds4_tq_test PRIVATE STRATA_DSV41_TQ_CPU=1)# the same with the plain-FMA tile instead of the gfx12 WMMA one (what CUDA and gfx11 builds run)add_executable(ds4_tq_test_portable tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp- src/dsv41/ds4_iq_cpu.cpp tools/trellis/tqenc.cpp)+ src/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)target_include_directories(ds4_tq_test_portable PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)target_link_libraries(ds4_tq_test_portable PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)@@ -817,6 +817,18 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)if(STRATA_ENABLE_HIP)target_compile_options(ds4_tq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")endif()+ # the i-quant experts (ds4_iq.cu) against ggml's own decode of the same blocks + a double reference, device and CPU+ # lane, for every type the device takes: the gate a new type (IQ2_XS ..) has to pass before the engine uses it+ set_source_files_properties(tests/dsv41/iq_expert_test.cpp PROPERTIES LANGUAGE HIP)+ add_executable(ds4_iq_test tests/dsv41/iq_expert_test.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp+ src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)+ target_include_directories(ds4_iq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}+ ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)+ target_link_libraries(ds4_iq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)+ target_compile_definitions(ds4_iq_test PRIVATE STRATA_DSV41_TQ_CPU=1)+ if(STRATA_ENABLE_HIP)+ target_compile_options(ds4_iq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")+ endif()endif()if(STRATA_ENABLE_HIP)foreach(_t strata_dsv41_kernels strata-dsv41)diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cppindex a1ea5cf6e..d9116df59 100644--- a/src/dsv41/ds4_gg_cpu.cpp+++ b/src/dsv41/ds4_gg_cpu.cpp@@ -6,6 +6,7 @@#include <cmath>#include <cstdio>#include <cstdlib>+#include <cstring>#include <string>#if defined(STRATA_DSV41_TQ_CPU)@@ -125,6 +126,24 @@ void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, consvoid gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);}++// a row of any ggml block type -> BF16 on the host (the embedding table's reader; Q5_K / Q4_K / Q6_K / Q8_0 rows of a+// GGUF whose dense tensors are not BF16): ggml's own to_float, then the same round-to-nearest-even to BF16 the Q8_0 path uses+bool gg_row_to_bf16(int type, const uint8_t* row, int k, uint16_t* out) {+ init_once();+ const ggml_type_traits* t = ggml_get_type_traits((ggml_type) type);+ if (t == nullptr || t->to_float == nullptr) return false;+ static thread_local std::vector<float> f;+ f.resize((size_t) k);+ t->to_float(row, f.data(), k);+ for (int i = 0; i < k; ++i) {+ uint32_t u;+ std::memcpy(&u, &f[(size_t) i], 4);+ u += 0x7fffu + ((u >> 16) & 1u);+ out[i] = (uint16_t) (u >> 16);+ }+ return true;+}} // namespace strata::ds4::gguf#elsenamespace strata::ds4::gguf {@@ -136,5 +155,6 @@ bool gg_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {void gg_cpu_quant(int, const float*, int, void*) {}void gg_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}void gg_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}+bool gg_row_to_bf16(int, const uint8_t*, int, uint16_t*) { return false; }} // namespace strata::ds4::gguf#endifdiff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hppindex 9eb00f810..d983a1dde 100644--- a/src/dsv41/ds4_gguf_cpu.hpp+++ b/src/dsv41/ds4_gguf_cpu.hpp@@ -48,6 +48,9 @@ void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int/// ggml block types on the CPU lane through ggml-cpu's row dots (ds4_gg_cpu.cpp), for the IQ / K-quant experts.bool gg_cpu_type(int type);+/// One row of a GGUF block type (Q4_K / Q5_K / Q6_K / Q8_0 / any type ggml has a to_float for) -> k values as BF16+/// (round to nearest even), the host embedding's format. False: ggml has no decode for this type.+bool gg_row_to_bf16(int type, const uint8_t* row, int k, uint16_t* out);bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);void gg_cpu_quant(int act, const float* x, int k, void* dst);void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex e99c1dcb5..e08bd2d35 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -28,6 +28,7 @@ constexpr int kWarps = 8, kThreads = kWarps * 32;template <int T>struct Ty;template <> struct Ty<kIQ2_XXS> { static constexpr int B = 66, TAB = 512, MIN = 0; };+template <> struct Ty<kIQ2_XS> { static constexpr int B = 74, TAB = 1024, MIN = 0; };template <> struct Ty<kIQ2_S> { static constexpr int B = 82, TAB = 2048, MIN = 0; };template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN = 0; };template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };@@ -43,7 +44,8 @@ template <> struct Ty<kQ6_K> { static constexpr int B = 210, TAB = 0, MIN = 0; }constexpr int kMXFP4 = 39;template <> struct Ty<kMXFP4> { static constexpr int B = 17, TAB = 0, MIN = 0; };template <int T> constexpr int kQK = T == kMXFP4 ? 32 : 256; // values a block-static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_s) == 82 && sizeof(block_iq3_xxs) == 98 &&+static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_xs) == 74 && sizeof(block_iq2_s) == 82 &&+ sizeof(block_iq3_xxs) == 98 &&sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 && sizeof(block_q4_K) == 144 &&sizeof(block_q2_K) == 84 && sizeof(block_q3_K) == 110,"ggml's block sizes");@@ -61,6 +63,7 @@ __device__ __forceinline__ void load_tab(uint32_t* tab) {}} else if constexpr (Ty<T>::TAB > 0) {const uint32_t* src = T == kIQ2_XXS ? (const uint32_t*) iq2xxs_grid+ : T == kIQ2_XS ? (const uint32_t*) iq2xs_grid: T == kIQ2_S ? (const uint32_t*) iq2s_grid: T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid: (const uint32_t*) iq3s_grid;@@ -149,6 +152,21 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,}s.s0 = s.s1 = 2 * (int) (a1 >> 28) + 1;s.d = h2f(b) * 0.125f;+ } else if constexpr (T == kIQ2_XS) { // d, qs[32] u16 (9-bit grid index, 7-bit sign index), scales[8] (two 4-bit)+ const uint8_t* q = b + 2 + 8 * ib;+ const uint32_t a0 = ld2(q), a1 = ld2(q + 4);+ const uint32_t v[4] = {a0 & 0xffffu, a0 >> 16, a1 & 0xffffu, a1 >> 16};+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint2 g = ((const uint2*) tab)[v[l] & 511u];+ const uint32_t sg = ksigns8(v[l] >> 9);+ s.q[2 * l] = sgn4(g.x, sg);+ s.q[2 * l + 1] = sgn4(g.y, sg >> 4);+ }+ const uint8_t sc = b[66 + ib];+ s.s0 = 2 * (int) (sc & 15) + 1;+ s.s1 = 2 * (int) (sc >> 4) + 1;+ s.d = h2f(b) * 0.125f;} else if constexpr (T == kIQ2_S) {const uint8_t* qs = b + 2;const uint32_t i4 = ld2(qs + 4 * ib), sg4 = ld2(qs + 32 + 4 * ib), h = b[66 + ib], sc = b[74 + ib];@@ -300,6 +318,17 @@ __device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, inconst uint2 g = ((const uint2*) tab)[q[p]];const uint32_t v = (a1 >> (7 * p)) & 127u, sg = v | ((__popc(v) & 1u) << 7);const float dl = h2f(b) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;+ }+ } else if constexpr (T == kIQ2_XS) {+ const uint16_t v = ((const uint16_t*) (b + 2 + 8 * ib))[p];+ const uint2 g = ((const uint2*) tab)[v & 511u];+ const uint32_t sg = ksigns8(v >> 9);+ const uint8_t sc = b[66 + ib];+ const float dl = h2f(b) * (float) (2 * (int) (p < 2 ? sc & 15 : sc >> 4) + 1) * 0.125f;#pragma unrollfor (int j = 0; j < 8; ++j) {const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);@@ -819,6 +848,7 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps#define DS4_IQ_SWITCH(t, F) \switch (t) { \case kIQ2_XXS: F(kIQ2_XXS); break; \+ case kIQ2_XS: F(kIQ2_XS); break; \case kIQ2_S: F(kIQ2_S); break; \case kIQ3_XXS: F(kIQ3_XXS); break; \case kIQ3_S: F(kIQ3_S); break; \@@ -945,6 +975,7 @@ int block_bytes(int t) {if (is_tq(t)) return 2 + 4 * tq_kbits(t);switch (t) {case kIQ2_XXS: return Ty<kIQ2_XXS>::B;+ case kIQ2_XS: return Ty<kIQ2_XS>::B;case kIQ2_S: return Ty<kIQ2_S>::B;case kIQ3_XXS: return Ty<kIQ3_XXS>::B;case kIQ3_S: return Ty<kIQ3_S>::B;@@ -967,6 +998,7 @@ const char* name(int t) {case kTQK5T: return "TQK5T";case kTQK4T: return "TQK4T";case kIQ2_XXS: return "IQ2_XXS";+ case kIQ2_XS: return "IQ2_XS";case kIQ2_S: return "IQ2_S";case kIQ3_XXS: return "IQ3_XXS";case kIQ3_S: return "IQ3_S";diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 3f0c8f52b..9a93c7e07 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -20,8 +20,8 @@ namespace strata::ds4::gguf::iq {// ggml type idsconstexpr int kQ5_K = 13, kQ6_K = 14; // (the dense matrices' K-quants, GEMV only)-constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,- kIQ4_XS = 23;+constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ2_XS = 17, kIQ3_XXS = 18, kIQ3_S = 21,+ kIQ2_S = 22, kIQ4_XS = 23;/// The trellis types (Strata's, ds4_tq.cu): ggml's TQK6 / TQK7 (145 / 146, the table codebook) and the experimental/// TQK5X / TQK4X (224 / 225, computed codebook) and TQK5T / TQK4T (226 / 227, table): a block of 128 values isdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex e4adb8ad8..72979866c 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1527,7 +1527,7 @@ struct Model::Impl {// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)static int gg_type_of(const std::string& dtype) {- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})if (dtype == gg::type_name(t)) return t;return -1;}@@ -2264,12 +2264,29 @@ struct Model::Impl {const Tensor* t = pk.get("embed.weight");const size_t nv = (size_t) g.vocab * g.dim;const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;- if (t == nullptr || (!q8 && (t->dtype != "BF16" || t->bytes != nv * 2))) {- err = "embed.weight: not a BF16 (or Q8_0) [vocab][dim] table";+ // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):+ // each row through ggml's to_float, to the same BF16 the host table holds+ const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;+ const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;+ const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;+ if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {+ err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";return false;}if (!halloc(embed_pinned, nv, err)) return false;- if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)+ if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)+ std::vector<uint8_t> row(erb);+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {+ if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {+ err = "embed.weight: read failed";+ return false;+ }+ if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {+ err = "embed.weight: ggml has no decode for " + t->dtype;+ return false;+ }+ }+ } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);for (size_t r = 0; r < (size_t) g.vocab; ++r) {if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {diff --git a/tests/dsv41/gg_dot_bench.cpp b/tests/dsv41/gg_dot_bench.cppindex 7e419a3a1..b8012e49a 100644--- a/tests/dsv41/gg_dot_bench.cpp+++ b/tests/dsv41/gg_dot_bench.cpp@@ -101,10 +101,12 @@ int main(int argc, char** argv) {const double t_gg = pass(0, p0.data());const double t_own = pass(1, p1.data());long bad = 0, differ = 0;- double worst = 0;+ double worst = 0, worst_ref = 0, absmax = 0, absdiff = 0;for (int t = 0; t < nt; ++t)for (long r = 0; r < rows; ++r) {const float x = o0[(size_t) t][(size_t) r], y = o1[(size_t) t][(size_t) r];+ absmax = std::max(absmax, (double) std::fabs(x));+ absdiff = std::max(absdiff, (double) std::fabs((double) x - y));if (std::memcmp(&x, &y, 4) != 0) {++differ;const double rel = std::fabs((double) x - y) / (std::fabs((double) x) + 1e-3);@@ -112,6 +114,40 @@ int main(int argc, char** argv) {if (rel > 1e-5) ++bad;}}+ // The pass criterion above divides by one value's magnitude, so a token whose dot cancels to near zero fails it while+ // agreeing to a few ulps. The real bound is the L1 norm of the products (|x - ref| <= eps * sum|w x|): decode a+ // sample of rows with ggml's own to_float and the activation as its Q8_K blocks hold it, and compare in double.+ int sampled = 0;+ {+ const ggml_type_traits* tt = ggml_get_type_traits((ggml_type) type);+ std::vector<float> wrow((size_t) n);+ // the activations (Q8_K: float d, int8 qs[256], int16 bsums[16]) as the rows see them+ std::vector<std::vector<float>> xv((size_t) nt, std::vector<float>((size_t) n));+ for (int t = 0; t < nt; ++t)+ for (int b = 0; b < n / 256; ++b) {+ const uint8_t* p = (const uint8_t*) ap[(size_t) t] + (size_t) b * (f.act_bytes / (size_t) (n / 256));+ float d;+ std::memcpy(&d, p, 4);+ const int8_t* q = (const int8_t*) (p + 4);+ for (int j = 0; j < 256; ++j) xv[(size_t) t][(size_t) b * 256 + j] = d * (float) q[j];+ }+ const long step = std::max(1L, rows / 64);+ for (long r = 0; r < rows && sampled < 64; r += step, ++sampled) {+ if (tt == nullptr || tt->to_float == nullptr) break;+ tt->to_float(w.data() + (size_t) r * rb, wrow.data(), n);+ for (int t = 0; t < nt; ++t) {+ double ref = 0, l1 = 0;+ for (int j = 0; j < n; ++j) {+ const double pr = (double) wrow[(size_t) j] * (double) xv[(size_t) t][(size_t) j];+ ref += pr;+ l1 += std::fabs(pr);+ }+ const double g = o0[(size_t) t][(size_t) r], o = o1[(size_t) t][(size_t) r];+ worst_ref = std::max(worst_ref, std::fabs(g - ref) / (l1 + 1e-30));+ worst_ref = std::max(worst_ref, std::fabs(o - ref) / (l1 + 1e-30));+ }+ }+ }// memory ceiling: a plain read of the same bytes on the same threadsdouble t_rd = 1e30;for (int r = 0; r < reps + 1; ++r) {@@ -132,8 +168,10 @@ int main(int argc, char** argv) {std::printf("type %d nt %d threads %d: %.2f GB of weights (%ld rows)\n", type, nt, threads, gb, rows);std::printf(" ggml-cpu rows %.3f s = %.1f GB/s | Strata AVX2 rows %.3f s = %.1f GB/s (x%.2f) | plain read %.1f GB/s\n", t_gg, gb / t_gg,t_own, gb / t_own, t_gg / t_own, gb / t_rd);- std::printf(" results: %ld of %ld differ in bits, worst relative %.2e, %ld beyond 1e-5\n", differ, (long) nt * rows, worst, bad);- if (bad != 0 || (exact && differ != 0)) {+ std::printf(" results: %ld of %ld differ in bits, worst |x-y|/|x| %.2e, %ld beyond 1e-5\n", differ, (long) nt * rows, worst, bad);+ std::printf(" vs a double reference on %d sampled rows: worst %.2e of the L1 norm (tol 1e-5); |x-y| max %.2e of |x| max %.2e\n",+ sampled, worst_ref, absdiff, absmax);+ if (worst_ref > 1e-5 || absdiff > 1e-6 * absmax + 1e-30 || (exact && differ != 0)) {std::printf("FAIL\n");return 1;}diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cppnew file mode 100644index 000000000..ff989087e--- /dev/null+++ b/tests/dsv41/iq_expert_test.cpp@@ -0,0 +1,305 @@+// Strata. The i-quant routed experts of the DeepSeek-V4.1 path (ds4_iq.cu on the device, ds4_iq_cpu.cpp on the CPU lane)+// against ggml's own decode of the same bytes plus a double reference: the token path (gate/up, down), the window path+// (grouped entries), the prompt sets (gate/up and down), and the CPU lane on the same experts. Random legal blocks: any+// bit pattern with a sane F16 scale is one. This is the H2 gate for a new type (IQ2_XS first): the kernel's integer sums+// must equal ggml's weights times the same activations.+// iq_expert_test [--type 16,17,18,21,22,23] [--bench] [--iq256 0|1]+#include "../../src/dsv41/ds4_gguf_cpu.hpp"+#include "../../src/dsv41/ds4_iq.cuh"++#include <hip/hip_runtime.h>++#include "ggml.h" // ggml_get_type_traits(type)->to_float: ggml's decode of the same blocks++#include <algorithm>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <vector>++namespace iq = strata::ds4::gguf::iq;+namespace gg = strata::ds4::gguf;+#define CK(x) do { hipError_t e_ = (x); if (e_ != hipSuccess) { std::printf("HIP error %s at %d\n", hipGetErrorString(e_), __LINE__); std::exit(2); } } while (0)++template <typename T> static T* up(const std::vector<T>& v) {+ T* d; CK(hipMalloc(&d, std::max<size_t>(1, v.size() * sizeof(T)))); CK(hipMemcpy(d, v.data(), v.size() * sizeof(T), hipMemcpyHostToDevice)); return d;+}+template <typename T> static std::vector<T> down_(const T* d, size_t n) {+ std::vector<T> v(n); CK(hipMemcpy(v.data(), d, n * sizeof(T), hipMemcpyDeviceToHost)); return v;+}+static float h2f(uint16_t h) {+ const int s = h >> 15, e = (h >> 10) & 31, m = h & 1023;+ float f = e == 0 ? std::ldexp((float) m, -24) : std::ldexp(1.0f + m / 1024.0f, e - 15);+ return s ? -f : f;+}+static uint16_t f2h(float f) { // round to nearest even, normal range+ uint32_t u; std::memcpy(&u, &f, 4);+ const uint32_t s = (u >> 16) & 0x8000u; int e = (int) ((u >> 23) & 0xff) - 127 + 15; uint32_t m = u & 0x7fffffu;+ uint32_t hm = m >> 13; const uint32_t rem = m & 0x1fffu;+ uint32_t v = ((uint32_t) e << 10) | hm;+ if (rem > 0x1000u || (rem == 0x1000u && (v & 1))) ++v;+ return (uint16_t) (s | v);+}++struct Q8 { std::vector<int8_t> q; std::vector<float> d; std::vector<float> xhat; };+// the engine's q8_quant: sc = max|x| / 127 per 32, q = round-half-even(x / sc)+static Q8 quant(const float* x, int k) {+ Q8 r; r.q.resize(k); r.d.resize(k / 32); r.xhat.resize(k);+ for (int b = 0; b < k / 32; ++b) {+ float am = 0; for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(x[b * 32 + j]));+ const float sc = am / 127.0f; r.d[b] = sc;+ for (int j = 0; j < 32; ++j) { const int q = am > 0 ? (int) std::nearbyint(x[b * 32 + j] / sc) : 0; r.q[b * 32 + j] = (int8_t) q; r.xhat[b * 32 + j] = (float) q * sc; }+ }+ return r;+}+static double swiglu(double g, double u, double lim) {+ if (lim > 0) { g = std::min(g, lim); u = std::min(std::max(u, -lim), lim); }+ return g / (1.0 + std::exp(-g)) * u;+}+static double dotd(const float* w, const float* x, int k, double* sabs = nullptr) {+ double s = 0, sa = 0; for (int i = 0; i < k; ++i) { const double t = (double) w[i] * x[i]; s += t; sa += std::fabs(t); } if (sabs) *sabs = sa; return s;+}+// an activation buffer of a ggml type (Q8_K for the i-quant rows) decoded by ggml itself+static std::vector<float> act_values(int type, const void* act, int k) {+ std::vector<float> v((size_t) k);+ if (type == 15) { // Q8_K: float d, int8 qs[256], int16 bsums[16] - no to_float (an intermediate type)+ const int nb = k / 256;+ const size_t bs = ggml_row_size((ggml_type) type, k) / (size_t) nb;+ for (int b = 0; b < nb; ++b) {+ const uint8_t* p = (const uint8_t*) act + (size_t) b * bs;+ float d; std::memcpy(&d, p, 4);+ const int8_t* qs = (const int8_t*) (p + 4);+ for (int j = 0; j < 256; ++j) v[(size_t) b * 256 + j] = d * (float) qs[j];+ }+ return v;+ }+ const ggml_type_traits* t = ggml_get_type_traits((ggml_type) type);+ if (t == nullptr || t->to_float == nullptr) { std::printf("no to_float for activation type %d\n", type); std::exit(2); }+ t->to_float(act, v.data(), k);+ return v;+}++struct Mat { std::vector<uint8_t> bytes; std::vector<float> w; int rows = 0, k = 0; };+// random legal blocks of `type` for a rows x k matrix, decoded by ggml itself+static Mat make_mat(std::mt19937_64& rng, int type, int rows, int k) {+ Mat m; m.rows = rows; m.k = k;+ const int bv = iq::block_values(type), bb = iq::block_bytes(type);+ const size_t nblk = (size_t) rows * (k / bv);+ m.bytes.resize(nblk * (size_t) bb);+ for (auto& b : m.bytes) b = (uint8_t) rng();+ for (size_t b = 0; b < nblk; ++b) { // a sane F16 scale in the first two bytes (every type here has d at offset 0)+ const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));+ std::memcpy(&m.bytes[b * (size_t) bb], &d, 2);+ }+ m.w.resize((size_t) rows * k);+ const ggml_type_traits* tr = ggml_get_type_traits((ggml_type) type);+ if (tr == nullptr || tr->to_float == nullptr) { std::printf("ggml has no to_float for type %d\n", type); std::exit(2); }+ const size_t rb = (size_t) (k / bv) * (size_t) bb;+ for (int r = 0; r < rows; ++r) tr->to_float(&m.bytes[(size_t) r * rb], &m.w[(size_t) r * k], k);+ return m;+}++// one decode token of the DeepSeek shape (n_embd 5120, n_ff 2304, top-6 experts): gate/up + down launches, per-layer time+static void bench(int type) {+ const int E = 5120, FF = 2304, NE = 6;+ const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);+ std::mt19937_64 rng(5);+ std::vector<uint8_t> one(blob);+ for (auto& b : one) b = (uint8_t) rng();+ std::vector<unsigned long long> plan(NE);+ for (int e = 0; e < NE; ++e) { uint8_t* d; CK(hipMalloc(&d, blob)); CK(hipMemcpy(d, one.data(), blob, hipMemcpyHostToDevice)); plan[e] = (unsigned long long) d; }+ std::vector<int8_t> xq(E, 3), hq((size_t) NE * FF, 3); std::vector<float> xd(E / 32, 0.01f), hd((size_t) NE * FF / 32, 0.01f);+ iq::Args a{};+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];+ float *dh, *dp; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMalloc(&dp, (size_t) NE * E * 4)); a.h = dh; a.part = dp;+ hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);+ for (int rep = 0; rep < 5; ++rep) { iq::gate_up(a, NE, 1, 0); iq::down(a, NE, 1, 0); }+ CK(hipDeviceSynchronize());+ float ms_gu = 0, ms_dn = 0;+ for (int part = 0; part < 2; ++part) {+ hipEventRecord(e0);+ for (int rep = 0; rep < 100; ++rep) { if (part == 0) iq::gate_up(a, NE, 1, 0); else iq::down(a, NE, 1, 0); }+ hipEventRecord(e1); hipEventSynchronize(e1);+ float ms; hipEventElapsedTime(&ms, e0, e1);+ (part == 0 ? ms_gu : ms_dn) = ms / 100;+ }+ const double bytes = (double) NE * blob;+ std::printf("bench %-7s 6 experts x 40 layers: gate/up %.0f us, down %.0f us, layer %.0f us -> %.0f GB/s, %.2f Tw/s, 40 layers %.1f ms/token\n",+ iq::name(type), ms_gu * 1e3, ms_dn * 1e3, (ms_gu + ms_dn) * 1e3,+ bytes / ((ms_gu + ms_dn) * 1e-3) / 1e9,+ (double) NE * 3 * FF * E / ((ms_gu + ms_dn) * 1e-3) / 1e12, 40.0 * (ms_gu + ms_dn));+}++int main(int argc, char** argv) {+ std::vector<int> types = {16, 17, 22, 18, 21, 23};+ for (int i = 1; i < argc; ++i) {+ if (std::string(argv[i]) == "--bench") {+ std::vector<int> bt;+ if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }+ else bt = types;+ for (int t : bt) bench(t);+ return 0;+ } else if (std::string(argv[i]) == "--type" && i + 1 < argc) {+ types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; }+ } else if (std::string(argv[i]) == "--iq256" && i + 1 < argc) {+ gg::gg_cpu_set_iq256(std::atoi(argv[++i]));+ }+ }+ std::mt19937_64 rng(11);+ std::normal_distribution<float> nd(0.f, 1.f);+ int fails = 0;+ auto report = [&](const char* what, int type, double worst, double tol) {+ const bool ok = worst <= tol;+ std::printf("%-7s %-22s worst %.2e (tol %.0e) %s\n", iq::name(type), what, worst, tol, ok ? "ok" : "FAIL");+ fails += !ok;+ };+ for (int type : types) {+ if (!iq::supported(type)) { std::printf("%-7s not a device type here\n", iq::name(type)); ++fails; continue; }+ const int E = 512, FF = 256, NEXP = 3, T = 4;+ const float lim = 7.0f;+ const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);+ std::vector<Mat> G(NEXP), U(NEXP), D(NEXP);+ auto refh = [&](int e, int r, const float* xv, double& scale) {+ double sg, su;+ const double g = dotd(&G[e].w[(size_t) r * E], xv, E, &sg), u = dotd(&U[e].w[(size_t) r * E], xv, E, &su);+ scale = sg + su + 1e-30;+ return swiglu(g, u, lim);+ };+ std::vector<uint8_t> blobs((size_t) NEXP * blob, 0);+ for (int e = 0; e < NEXP; ++e) {+ G[e] = make_mat(rng, type, FF, E); U[e] = make_mat(rng, type, FF, E); D[e] = make_mat(rng, type, E, FF);+ std::memcpy(&blobs[e * blob + off[0]], G[e].bytes.data(), G[e].bytes.size());+ std::memcpy(&blobs[e * blob + off[1]], U[e].bytes.data(), U[e].bytes.size());+ std::memcpy(&blobs[e * blob + off[2]], D[e].bytes.data(), D[e].bytes.size());+ }+ uint8_t* dblob = up(blobs);+ std::vector<float> x((size_t) T * E);+ for (auto& v : x) v = nd(rng) * (1.0f + 3.0f * (rng() % 7 == 0));+ std::vector<Q8> qx(T);+ std::vector<int8_t> xq((size_t) T * E); std::vector<float> xd((size_t) T * E / 32);+ for (int t = 0; t < T; ++t) { qx[t] = quant(&x[(size_t) t * E], E); std::memcpy(&xq[(size_t) t * E], qx[t].q.data(), E); std::memcpy(&xd[(size_t) t * E / 32], qx[t].d.data(), E / 32 * 4); }+ const int8_t* dxq = up(xq); const float* dxd = up(xd);++ // ---- token path+ std::vector<float> href((size_t) NEXP * FF);+ std::vector<Q8> qh(NEXP);+ {+ std::vector<unsigned long long> plan(NEXP);+ for (int e = 0; e < NEXP; ++e) plan[e] = (unsigned long long) (dblob + e * blob);+ iq::Args a{};+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = lim; a.xq = dxq; a.xd = dxd; a.tg = a.td = type;+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];+ float* dh; CK(hipMalloc(&dh, (size_t) NEXP * FF * 4)); a.h = dh;+ if (!iq::gate_up(a, NEXP, 1, 0)) { std::printf("gate_up refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const auto h = down_(dh, (size_t) NEXP * FF);+ double worst = 0;+ for (int e = 0; e < NEXP; ++e)+ for (int r = 0; r < FF; ++r) {+ double sc;+ const double ref = refh(e, r, qx[0].xhat.data(), sc);+ href[(size_t) e * FF + r] = (float) ref;+ worst = std::max(worst, std::fabs(h[(size_t) e * FF + r] - ref) / sc);+ }+ report("token gate_up", type, worst, 1e-5);+ std::vector<int8_t> hq((size_t) NEXP * FF); std::vector<float> hd((size_t) NEXP * FF / 32);+ for (int e = 0; e < NEXP; ++e) { qh[e] = quant(&href[(size_t) e * FF], FF); std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }+ a.hq = up(hq); a.hd = up(hd);+ float* dp; CK(hipMalloc(&dp, (size_t) NEXP * E * 4)); a.part = dp;+ if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const auto part = down_(dp, (size_t) NEXP * E);+ worst = 0;+ for (int e = 0; e < NEXP; ++e)+ for (int r = 0; r < E; ++r) {+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], qh[e].xhat.data(), FF, &sa);+ worst = std::max(worst, std::fabs(part[(size_t) e * E + r] - ref) / (sa + 1e-30));+ }+ report("token down", type, worst, 1e-6);++ // ---- the prompt sets: expert 0 owns rows [0, 37), expert 2 [37, 42)+ const int NR = 42;+ std::vector<int> bounds = {0, 37, 37, 42}, idx(NR);+ for (int i = 0; i < NR; ++i) idx[i] = i % T;+ iq::SetArgs q{};+ q.wbase = dblob; q.stride = blob; for (int r = 0; r < 3; ++r) q.tr[r] = off[r];+ q.bounds = up(bounds); q.row0 = 0; q.max_rows = 37; q.n_exp = 3; q.n_embd = E; q.n_ff = FF; q.limit = lim;+ q.tg = q.td = type; q.xf = up(x); q.idx = up(idx); q.ldx = E;+ uint16_t* dh16; CK(hipMalloc(&dh16, (size_t) NR * FF * 2)); q.h = dh16; q.ldo = FF;+ if (!iq::set_gate_up(q, 0)) { std::printf("set_gate_up refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const auto h16 = down_(dh16, (size_t) NR * FF);+ double w2 = 0;+ for (int e = 0; e < 3; ++e)+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr)+ for (int r = 0; r < FF; ++r) {+ double sc;+ const double ref = refh(e, r, &x[(size_t) idx[lr] * E], sc);+ w2 = std::max(w2, std::fabs((double) h2f(h16[(size_t) lr * FF + r]) - ref) / sc);+ }+ report("set gate_up (f16)", type, w2, 1e-2);+ q.xh = dh16; q.ldx = FF;+ float* dy; CK(hipMalloc(&dy, (size_t) NR * E * 4)); q.y = dy; q.ldo = E;+ if (!iq::set_down(q, 0)) { std::printf("set_down refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const auto y = down_(dy, (size_t) NR * E);+ w2 = 0;+ for (int e = 0; e < 3; ++e)+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {+ std::vector<float> hf(FF);+ for (int c = 0; c < FF; ++c) hf[c] = h2f(h16[(size_t) lr * FF + c]);+ for (int r = 0; r < E; ++r) {+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hf.data(), FF, &sa);+ w2 = std::max(w2, std::fabs(y[(size_t) lr * E + r] - ref) / (sa + 1e-30));+ }+ }+ // (the prompt sets' tiles take F16 operands for the i-quants: the tile's own rounding, not the token path's)+ report("set down", type, w2, 3e-3);+ }++ // ---- the CPU lane on the same experts and inputs (ggml-cpu's rows, or Strata's AVX2 ones with DS4_GG_IQ256=1)+ {+ gg::IqCpu fm; std::string err;+ if (!gg::iq_cpu_fmt(type, type, E, FF, fm, err)) { std::printf("%-7s cpu lane: %s\n", iq::name(type), err.c_str()); ++fails; }+ else {+ std::vector<uint8_t> act(fm.act_bytes), hact(fm.h_bytes);+ gg::iq_cpu_quant(fm.gu_act, x.data(), E, act.data());+ // (the CPU lane rounds the activation to its own type - Q8_K, one F32 scale per 256 values - so the+ // reference is that quantized activation, not the device's per-32 one)+ const std::vector<float> xhat = act_values(fm.gu_act, act.data(), E);+ double wg = 0, wd = 0;+ for (int e = 0; e < NEXP; ++e) {+ std::vector<float> hc(FF), yc(E);+ float* hp[1] = {hc.data()}; float* yp[1] = {yc.data()};+ const void* ap[1] = {act.data()};+ gg::iq_cpu_gate_up(fm, &blobs[e * blob + off[0]], &blobs[e * blob + off[1]], ap, 1, hp, 0, FF, lim);+ for (int r = 0; r < FF; ++r) {+ double sg, su;+ const double g = dotd(&G[e].w[(size_t) r * E], xhat.data(), E, &sg);+ const double u = dotd(&U[e].w[(size_t) r * E], xhat.data(), E, &su);+ wg = std::max(wg, std::fabs(hc[r] - swiglu(g, u, lim)) / (sg + su + 1e-30));+ }+ gg::iq_cpu_quant(fm.d_act, &href[(size_t) e * FF], FF, hact.data());+ const std::vector<float> hhat = act_values(fm.d_act, hact.data(), FF);+ const void* hpp[1] = {hact.data()};+ gg::iq_cpu_down(fm, &blobs[e * blob + off[2]], hpp, 1, yp, 0, E);+ for (int r = 0; r < E; ++r) { double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hhat.data(), FF, &sa); wd = std::max(wd, std::fabs(yc[r] - ref) / (sa + 1e-30)); }+ }+ report("cpu gate_up vs ref", type, wg, 1e-5);+ report("cpu down vs ref", type, wd, 1e-5);+ std::printf("%-7s cpu rows: %s\n", iq::name(type), gg::iq_cpu_own_rows(type) ? "Strata AVX2" : "ggml-cpu");+ }+ }+ CK(hipFree(dblob));+ }+ std::printf("%s\n", fails ? "FAILED" : "ALL OK");+ return fails;+}