Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: IQ2_XS device kernels (bit-exact vs ggml), Q5_K/Q4_K/Q6_K embedding rows on the host, i-quant expert test, CPU-lane bench reference

Niko122149c157fae318
diff --git a/CMakeLists.txt b/CMakeLists.txt
index bcf1734e4..ff5409244 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Strata Engine - top-level build (P1.S1).
+# Strata Engine - top-level build (P1.S1).
#
# Targets (phase-1 §P1.S1): `strata_artifact` (no CUDA dependency), `tests`. `strata_plan` and the C++
# `strata-pack` are added when their sources exist - a target with no sources would make the build graph
@@ -799,14 +799,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
# the trellis experts (device kernels ds4_tq.cu, CPU lane ds4_tq_cpu.cpp) against a double reference; needs a GPU, no model
set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE HIP)
add_executable(ds4_tq_test tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp
- src/dsv41/ds4_iq_cpu.cpp tools/trellis/tqenc.cpp)
+ src/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)
target_include_directories(ds4_tq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)
target_link_libraries(ds4_tq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)
target_compile_definitions(ds4_tq_test PRIVATE STRATA_DSV41_TQ_CPU=1)
# the same with the plain-FMA tile instead of the gfx12 WMMA one (what CUDA and gfx11 builds run)
add_executable(ds4_tq_test_portable tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp
- src/dsv41/ds4_iq_cpu.cpp tools/trellis/tqenc.cpp)
+ src/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)
target_include_directories(ds4_tq_test_portable PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)
target_link_libraries(ds4_tq_test_portable PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)
@@ -817,6 +817,18 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
if(STRATA_ENABLE_HIP)
target_compile_options(ds4_tq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
endif()
+ # the i-quant experts (ds4_iq.cu) against ggml's own decode of the same blocks + a double reference, device and CPU
+ # lane, for every type the device takes: the gate a new type (IQ2_XS ..) has to pass before the engine uses it
+ set_source_files_properties(tests/dsv41/iq_expert_test.cpp PROPERTIES LANGUAGE HIP)
+ add_executable(ds4_iq_test tests/dsv41/iq_expert_test.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
+ src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)
+ target_include_directories(ds4_iq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
+ ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)
+ target_link_libraries(ds4_iq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)
+ target_compile_definitions(ds4_iq_test PRIVATE STRATA_DSV41_TQ_CPU=1)
+ if(STRATA_ENABLE_HIP)
+ target_compile_options(ds4_iq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
+ endif()
endif()
if(STRATA_ENABLE_HIP)
foreach(_t strata_dsv41_kernels strata-dsv41)
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cpp
index a1ea5cf6e..d9116df59 100644
--- a/src/dsv41/ds4_gg_cpu.cpp
+++ b/src/dsv41/ds4_gg_cpu.cpp
@@ -6,6 +6,7 @@
#include <cmath>
#include <cstdio>
#include <cstdlib>
+#include <cstring>
#include <string>
#if defined(STRATA_DSV41_TQ_CPU)
@@ -125,6 +126,24 @@ void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, cons
void gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);
}
+
+// a row of any ggml block type -> BF16 on the host (the embedding table's reader; Q5_K / Q4_K / Q6_K / Q8_0 rows of a
+// GGUF whose dense tensors are not BF16): ggml's own to_float, then the same round-to-nearest-even to BF16 the Q8_0 path uses
+bool gg_row_to_bf16(int type, const uint8_t* row, int k, uint16_t* out) {
+ init_once();
+ const ggml_type_traits* t = ggml_get_type_traits((ggml_type) type);
+ if (t == nullptr || t->to_float == nullptr) return false;
+ static thread_local std::vector<float> f;
+ f.resize((size_t) k);
+ t->to_float(row, f.data(), k);
+ for (int i = 0; i < k; ++i) {
+ uint32_t u;
+ std::memcpy(&u, &f[(size_t) i], 4);
+ u += 0x7fffu + ((u >> 16) & 1u);
+ out[i] = (uint16_t) (u >> 16);
+ }
+ return true;
+}
} // namespace strata::ds4::gguf
#else
namespace strata::ds4::gguf {
@@ -136,5 +155,6 @@ bool gg_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
void gg_cpu_quant(int, const float*, int, void*) {}
void gg_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}
void gg_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
+bool gg_row_to_bf16(int, const uint8_t*, int, uint16_t*) { return false; }
} // namespace strata::ds4::gguf
#endif
diff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hpp
index 9eb00f810..d983a1dde 100644
--- a/src/dsv41/ds4_gguf_cpu.hpp
+++ b/src/dsv41/ds4_gguf_cpu.hpp
@@ -48,6 +48,9 @@ void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int
/// ggml block types on the CPU lane through ggml-cpu's row dots (ds4_gg_cpu.cpp), for the IQ / K-quant experts.
bool gg_cpu_type(int type);
+/// One row of a GGUF block type (Q4_K / Q5_K / Q6_K / Q8_0 / any type ggml has a to_float for) -> k values as BF16
+/// (round to nearest even), the host embedding's format. False: ggml has no decode for this type.
+bool gg_row_to_bf16(int type, const uint8_t* row, int k, uint16_t* out);
bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);
void gg_cpu_quant(int act, const float* x, int k, void* dst);
void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index e99c1dcb5..e08bd2d35 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -28,6 +28,7 @@ constexpr int kWarps = 8, kThreads = kWarps * 32;
template <int T>
struct Ty;
template <> struct Ty<kIQ2_XXS> { static constexpr int B = 66, TAB = 512, MIN = 0; };
+template <> struct Ty<kIQ2_XS> { static constexpr int B = 74, TAB = 1024, MIN = 0; };
template <> struct Ty<kIQ2_S> { static constexpr int B = 82, TAB = 2048, MIN = 0; };
template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN = 0; };
template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };
@@ -43,7 +44,8 @@ template <> struct Ty<kQ6_K> { static constexpr int B = 210, TAB = 0, MIN = 0; }
constexpr int kMXFP4 = 39;
template <> struct Ty<kMXFP4> { static constexpr int B = 17, TAB = 0, MIN = 0; };
template <int T> constexpr int kQK = T == kMXFP4 ? 32 : 256; // values a block
-static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_s) == 82 && sizeof(block_iq3_xxs) == 98 &&
+static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_xs) == 74 && sizeof(block_iq2_s) == 82 &&
+ sizeof(block_iq3_xxs) == 98 &&
sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 && sizeof(block_q4_K) == 144 &&
sizeof(block_q2_K) == 84 && sizeof(block_q3_K) == 110,
"ggml's block sizes");
@@ -61,6 +63,7 @@ __device__ __forceinline__ void load_tab(uint32_t* tab) {
}
} else if constexpr (Ty<T>::TAB > 0) {
const uint32_t* src = T == kIQ2_XXS ? (const uint32_t*) iq2xxs_grid
+ : T == kIQ2_XS ? (const uint32_t*) iq2xs_grid
: T == kIQ2_S ? (const uint32_t*) iq2s_grid
: T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid
: (const uint32_t*) iq3s_grid;
@@ -149,6 +152,21 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,
}
s.s0 = s.s1 = 2 * (int) (a1 >> 28) + 1;
s.d = h2f(b) * 0.125f;
+ } else if constexpr (T == kIQ2_XS) { // d, qs[32] u16 (9-bit grid index, 7-bit sign index), scales[8] (two 4-bit)
+ const uint8_t* q = b + 2 + 8 * ib;
+ const uint32_t a0 = ld2(q), a1 = ld2(q + 4);
+ const uint32_t v[4] = {a0 & 0xffffu, a0 >> 16, a1 & 0xffffu, a1 >> 16};
+#pragma unroll
+ for (int l = 0; l < 4; ++l) {
+ const uint2 g = ((const uint2*) tab)[v[l] & 511u];
+ const uint32_t sg = ksigns8(v[l] >> 9);
+ s.q[2 * l] = sgn4(g.x, sg);
+ s.q[2 * l + 1] = sgn4(g.y, sg >> 4);
+ }
+ const uint8_t sc = b[66 + ib];
+ s.s0 = 2 * (int) (sc & 15) + 1;
+ s.s1 = 2 * (int) (sc >> 4) + 1;
+ s.d = h2f(b) * 0.125f;
} else if constexpr (T == kIQ2_S) {
const uint8_t* qs = b + 2;
const uint32_t i4 = ld2(qs + 4 * ib), sg4 = ld2(qs + 32 + 4 * ib), h = b[66 + ib], sc = b[74 + ib];
@@ -300,6 +318,17 @@ __device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, in
const uint2 g = ((const uint2*) tab)[q[p]];
const uint32_t v = (a1 >> (7 * p)) & 127u, sg = v | ((__popc(v) & 1u) << 7);
const float dl = h2f(b) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
+ }
+ } else if constexpr (T == kIQ2_XS) {
+ const uint16_t v = ((const uint16_t*) (b + 2 + 8 * ib))[p];
+ const uint2 g = ((const uint2*) tab)[v & 511u];
+ const uint32_t sg = ksigns8(v >> 9);
+ const uint8_t sc = b[66 + ib];
+ const float dl = h2f(b) * (float) (2 * (int) (p < 2 ? sc & 15 : sc >> 4) + 1) * 0.125f;
#pragma unroll
for (int j = 0; j < 8; ++j) {
const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
@@ -819,6 +848,7 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps
#define DS4_IQ_SWITCH(t, F) \
switch (t) { \
case kIQ2_XXS: F(kIQ2_XXS); break; \
+ case kIQ2_XS: F(kIQ2_XS); break; \
case kIQ2_S: F(kIQ2_S); break; \
case kIQ3_XXS: F(kIQ3_XXS); break; \
case kIQ3_S: F(kIQ3_S); break; \
@@ -945,6 +975,7 @@ int block_bytes(int t) {
if (is_tq(t)) return 2 + 4 * tq_kbits(t);
switch (t) {
case kIQ2_XXS: return Ty<kIQ2_XXS>::B;
+ case kIQ2_XS: return Ty<kIQ2_XS>::B;
case kIQ2_S: return Ty<kIQ2_S>::B;
case kIQ3_XXS: return Ty<kIQ3_XXS>::B;
case kIQ3_S: return Ty<kIQ3_S>::B;
@@ -967,6 +998,7 @@ const char* name(int t) {
case kTQK5T: return "TQK5T";
case kTQK4T: return "TQK4T";
case kIQ2_XXS: return "IQ2_XXS";
+ case kIQ2_XS: return "IQ2_XS";
case kIQ2_S: return "IQ2_S";
case kIQ3_XXS: return "IQ3_XXS";
case kIQ3_S: return "IQ3_S";
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index 3f0c8f52b..9a93c7e07 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -20,8 +20,8 @@ namespace strata::ds4::gguf::iq {
// ggml type ids
constexpr int kQ5_K = 13, kQ6_K = 14; // (the dense matrices' K-quants, GEMV only)
-constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,
- kIQ4_XS = 23;
+constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ2_XS = 17, kIQ3_XXS = 18, kIQ3_S = 21,
+ kIQ2_S = 22, kIQ4_XS = 23;
/// The trellis types (Strata's, ds4_tq.cu): ggml's TQK6 / TQK7 (145 / 146, the table codebook) and the experimental
/// TQK5X / TQK4X (224 / 225, computed codebook) and TQK5T / TQK4T (226 / 227, table): a block of 128 values is
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index e4adb8ad8..72979866c 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1527,7 +1527,7 @@ struct Model::Impl {
// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
static int gg_type_of(const std::string& dtype) {
- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})
+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})
if (dtype == gg::type_name(t)) return t;
return -1;
}
@@ -2264,12 +2264,29 @@ struct Model::Impl {
const Tensor* t = pk.get("embed.weight");
const size_t nv = (size_t) g.vocab * g.dim;
const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;
- if (t == nullptr || (!q8 && (t->dtype != "BF16" || t->bytes != nv * 2))) {
- err = "embed.weight: not a BF16 (or Q8_0) [vocab][dim] table";
+ // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):
+ // each row through ggml's to_float, to the same BF16 the host table holds
+ const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;
+ const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;
+ const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;
+ if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {
+ err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";
return false;
}
if (!halloc(embed_pinned, nv, err)) return false;
- if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)
+ if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)
+ std::vector<uint8_t> row(erb);
+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {
+ if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {
+ err = "embed.weight: ggml has no decode for " + t->dtype;
+ return false;
+ }
+ }
+ } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)
std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);
for (size_t r = 0; r < (size_t) g.vocab; ++r) {
if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {
diff --git a/tests/dsv41/gg_dot_bench.cpp b/tests/dsv41/gg_dot_bench.cpp
index 7e419a3a1..b8012e49a 100644
--- a/tests/dsv41/gg_dot_bench.cpp
+++ b/tests/dsv41/gg_dot_bench.cpp
@@ -101,10 +101,12 @@ int main(int argc, char** argv) {
const double t_gg = pass(0, p0.data());
const double t_own = pass(1, p1.data());
long bad = 0, differ = 0;
- double worst = 0;
+ double worst = 0, worst_ref = 0, absmax = 0, absdiff = 0;
for (int t = 0; t < nt; ++t)
for (long r = 0; r < rows; ++r) {
const float x = o0[(size_t) t][(size_t) r], y = o1[(size_t) t][(size_t) r];
+ absmax = std::max(absmax, (double) std::fabs(x));
+ absdiff = std::max(absdiff, (double) std::fabs((double) x - y));
if (std::memcmp(&x, &y, 4) != 0) {
++differ;
const double rel = std::fabs((double) x - y) / (std::fabs((double) x) + 1e-3);
@@ -112,6 +114,40 @@ int main(int argc, char** argv) {
if (rel > 1e-5) ++bad;
}
}
+ // The pass criterion above divides by one value's magnitude, so a token whose dot cancels to near zero fails it while
+ // agreeing to a few ulps. The real bound is the L1 norm of the products (|x - ref| <= eps * sum|w x|): decode a
+ // sample of rows with ggml's own to_float and the activation as its Q8_K blocks hold it, and compare in double.
+ int sampled = 0;
+ {
+ const ggml_type_traits* tt = ggml_get_type_traits((ggml_type) type);
+ std::vector<float> wrow((size_t) n);
+ // the activations (Q8_K: float d, int8 qs[256], int16 bsums[16]) as the rows see them
+ std::vector<std::vector<float>> xv((size_t) nt, std::vector<float>((size_t) n));
+ for (int t = 0; t < nt; ++t)
+ for (int b = 0; b < n / 256; ++b) {
+ const uint8_t* p = (const uint8_t*) ap[(size_t) t] + (size_t) b * (f.act_bytes / (size_t) (n / 256));
+ float d;
+ std::memcpy(&d, p, 4);
+ const int8_t* q = (const int8_t*) (p + 4);
+ for (int j = 0; j < 256; ++j) xv[(size_t) t][(size_t) b * 256 + j] = d * (float) q[j];
+ }
+ const long step = std::max(1L, rows / 64);
+ for (long r = 0; r < rows && sampled < 64; r += step, ++sampled) {
+ if (tt == nullptr || tt->to_float == nullptr) break;
+ tt->to_float(w.data() + (size_t) r * rb, wrow.data(), n);
+ for (int t = 0; t < nt; ++t) {
+ double ref = 0, l1 = 0;
+ for (int j = 0; j < n; ++j) {
+ const double pr = (double) wrow[(size_t) j] * (double) xv[(size_t) t][(size_t) j];
+ ref += pr;
+ l1 += std::fabs(pr);
+ }
+ const double g = o0[(size_t) t][(size_t) r], o = o1[(size_t) t][(size_t) r];
+ worst_ref = std::max(worst_ref, std::fabs(g - ref) / (l1 + 1e-30));
+ worst_ref = std::max(worst_ref, std::fabs(o - ref) / (l1 + 1e-30));
+ }
+ }
+ }
// memory ceiling: a plain read of the same bytes on the same threads
double t_rd = 1e30;
for (int r = 0; r < reps + 1; ++r) {
@@ -132,8 +168,10 @@ int main(int argc, char** argv) {
std::printf("type %d nt %d threads %d: %.2f GB of weights (%ld rows)\n", type, nt, threads, gb, rows);
std::printf(" ggml-cpu rows %.3f s = %.1f GB/s | Strata AVX2 rows %.3f s = %.1f GB/s (x%.2f) | plain read %.1f GB/s\n", t_gg, gb / t_gg,
t_own, gb / t_own, t_gg / t_own, gb / t_rd);
- std::printf(" results: %ld of %ld differ in bits, worst relative %.2e, %ld beyond 1e-5\n", differ, (long) nt * rows, worst, bad);
- if (bad != 0 || (exact && differ != 0)) {
+ std::printf(" results: %ld of %ld differ in bits, worst |x-y|/|x| %.2e, %ld beyond 1e-5\n", differ, (long) nt * rows, worst, bad);
+ std::printf(" vs a double reference on %d sampled rows: worst %.2e of the L1 norm (tol 1e-5); |x-y| max %.2e of |x| max %.2e\n",
+ sampled, worst_ref, absdiff, absmax);
+ if (worst_ref > 1e-5 || absdiff > 1e-6 * absmax + 1e-30 || (exact && differ != 0)) {
std::printf("FAIL\n");
return 1;
}
diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cpp
new file mode 100644
index 000000000..ff989087e
--- /dev/null
+++ b/tests/dsv41/iq_expert_test.cpp
@@ -0,0 +1,305 @@
+// Strata. The i-quant routed experts of the DeepSeek-V4.1 path (ds4_iq.cu on the device, ds4_iq_cpu.cpp on the CPU lane)
+// against ggml's own decode of the same bytes plus a double reference: the token path (gate/up, down), the window path
+// (grouped entries), the prompt sets (gate/up and down), and the CPU lane on the same experts. Random legal blocks: any
+// bit pattern with a sane F16 scale is one. This is the H2 gate for a new type (IQ2_XS first): the kernel's integer sums
+// must equal ggml's weights times the same activations.
+// iq_expert_test [--type 16,17,18,21,22,23] [--bench] [--iq256 0|1]
+#include "../../src/dsv41/ds4_gguf_cpu.hpp"
+#include "../../src/dsv41/ds4_iq.cuh"
+
+#include <hip/hip_runtime.h>
+
+#include "ggml.h" // ggml_get_type_traits(type)->to_float: ggml's decode of the same blocks
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace iq = strata::ds4::gguf::iq;
+namespace gg = strata::ds4::gguf;
+#define CK(x) do { hipError_t e_ = (x); if (e_ != hipSuccess) { std::printf("HIP error %s at %d\n", hipGetErrorString(e_), __LINE__); std::exit(2); } } while (0)
+
+template <typename T> static T* up(const std::vector<T>& v) {
+ T* d; CK(hipMalloc(&d, std::max<size_t>(1, v.size() * sizeof(T)))); CK(hipMemcpy(d, v.data(), v.size() * sizeof(T), hipMemcpyHostToDevice)); return d;
+}
+template <typename T> static std::vector<T> down_(const T* d, size_t n) {
+ std::vector<T> v(n); CK(hipMemcpy(v.data(), d, n * sizeof(T), hipMemcpyDeviceToHost)); return v;
+}
+static float h2f(uint16_t h) {
+ const int s = h >> 15, e = (h >> 10) & 31, m = h & 1023;
+ float f = e == 0 ? std::ldexp((float) m, -24) : std::ldexp(1.0f + m / 1024.0f, e - 15);
+ return s ? -f : f;
+}
+static uint16_t f2h(float f) { // round to nearest even, normal range
+ uint32_t u; std::memcpy(&u, &f, 4);
+ const uint32_t s = (u >> 16) & 0x8000u; int e = (int) ((u >> 23) & 0xff) - 127 + 15; uint32_t m = u & 0x7fffffu;
+ uint32_t hm = m >> 13; const uint32_t rem = m & 0x1fffu;
+ uint32_t v = ((uint32_t) e << 10) | hm;
+ if (rem > 0x1000u || (rem == 0x1000u && (v & 1))) ++v;
+ return (uint16_t) (s | v);
+}
+
+struct Q8 { std::vector<int8_t> q; std::vector<float> d; std::vector<float> xhat; };
+// the engine's q8_quant: sc = max|x| / 127 per 32, q = round-half-even(x / sc)
+static Q8 quant(const float* x, int k) {
+ Q8 r; r.q.resize(k); r.d.resize(k / 32); r.xhat.resize(k);
+ for (int b = 0; b < k / 32; ++b) {
+ float am = 0; for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(x[b * 32 + j]));
+ const float sc = am / 127.0f; r.d[b] = sc;
+ for (int j = 0; j < 32; ++j) { const int q = am > 0 ? (int) std::nearbyint(x[b * 32 + j] / sc) : 0; r.q[b * 32 + j] = (int8_t) q; r.xhat[b * 32 + j] = (float) q * sc; }
+ }
+ return r;
+}
+static double swiglu(double g, double u, double lim) {
+ if (lim > 0) { g = std::min(g, lim); u = std::min(std::max(u, -lim), lim); }
+ return g / (1.0 + std::exp(-g)) * u;
+}
+static double dotd(const float* w, const float* x, int k, double* sabs = nullptr) {
+ double s = 0, sa = 0; for (int i = 0; i < k; ++i) { const double t = (double) w[i] * x[i]; s += t; sa += std::fabs(t); } if (sabs) *sabs = sa; return s;
+}
+// an activation buffer of a ggml type (Q8_K for the i-quant rows) decoded by ggml itself
+static std::vector<float> act_values(int type, const void* act, int k) {
+ std::vector<float> v((size_t) k);
+ if (type == 15) { // Q8_K: float d, int8 qs[256], int16 bsums[16] - no to_float (an intermediate type)
+ const int nb = k / 256;
+ const size_t bs = ggml_row_size((ggml_type) type, k) / (size_t) nb;
+ for (int b = 0; b < nb; ++b) {
+ const uint8_t* p = (const uint8_t*) act + (size_t) b * bs;
+ float d; std::memcpy(&d, p, 4);
+ const int8_t* qs = (const int8_t*) (p + 4);
+ for (int j = 0; j < 256; ++j) v[(size_t) b * 256 + j] = d * (float) qs[j];
+ }
+ return v;
+ }
+ const ggml_type_traits* t = ggml_get_type_traits((ggml_type) type);
+ if (t == nullptr || t->to_float == nullptr) { std::printf("no to_float for activation type %d\n", type); std::exit(2); }
+ t->to_float(act, v.data(), k);
+ return v;
+}
+
+struct Mat { std::vector<uint8_t> bytes; std::vector<float> w; int rows = 0, k = 0; };
+// random legal blocks of `type` for a rows x k matrix, decoded by ggml itself
+static Mat make_mat(std::mt19937_64& rng, int type, int rows, int k) {
+ Mat m; m.rows = rows; m.k = k;
+ const int bv = iq::block_values(type), bb = iq::block_bytes(type);
+ const size_t nblk = (size_t) rows * (k / bv);
+ m.bytes.resize(nblk * (size_t) bb);
+ for (auto& b : m.bytes) b = (uint8_t) rng();
+ for (size_t b = 0; b < nblk; ++b) { // a sane F16 scale in the first two bytes (every type here has d at offset 0)
+ const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));
+ std::memcpy(&m.bytes[b * (size_t) bb], &d, 2);
+ }
+ m.w.resize((size_t) rows * k);
+ const ggml_type_traits* tr = ggml_get_type_traits((ggml_type) type);
+ if (tr == nullptr || tr->to_float == nullptr) { std::printf("ggml has no to_float for type %d\n", type); std::exit(2); }
+ const size_t rb = (size_t) (k / bv) * (size_t) bb;
+ for (int r = 0; r < rows; ++r) tr->to_float(&m.bytes[(size_t) r * rb], &m.w[(size_t) r * k], k);
+ return m;
+}
+
+// one decode token of the DeepSeek shape (n_embd 5120, n_ff 2304, top-6 experts): gate/up + down launches, per-layer time
+static void bench(int type) {
+ const int E = 5120, FF = 2304, NE = 6;
+ const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };
+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
+ std::mt19937_64 rng(5);
+ std::vector<uint8_t> one(blob);
+ for (auto& b : one) b = (uint8_t) rng();
+ std::vector<unsigned long long> plan(NE);
+ for (int e = 0; e < NE; ++e) { uint8_t* d; CK(hipMalloc(&d, blob)); CK(hipMemcpy(d, one.data(), blob, hipMemcpyHostToDevice)); plan[e] = (unsigned long long) d; }
+ std::vector<int8_t> xq(E, 3), hq((size_t) NE * FF, 3); std::vector<float> xd(E / 32, 0.01f), hd((size_t) NE * FF / 32, 0.01f);
+ iq::Args a{};
+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;
+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
+ float *dh, *dp; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMalloc(&dp, (size_t) NE * E * 4)); a.h = dh; a.part = dp;
+ hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);
+ for (int rep = 0; rep < 5; ++rep) { iq::gate_up(a, NE, 1, 0); iq::down(a, NE, 1, 0); }
+ CK(hipDeviceSynchronize());
+ float ms_gu = 0, ms_dn = 0;
+ for (int part = 0; part < 2; ++part) {
+ hipEventRecord(e0);
+ for (int rep = 0; rep < 100; ++rep) { if (part == 0) iq::gate_up(a, NE, 1, 0); else iq::down(a, NE, 1, 0); }
+ hipEventRecord(e1); hipEventSynchronize(e1);
+ float ms; hipEventElapsedTime(&ms, e0, e1);
+ (part == 0 ? ms_gu : ms_dn) = ms / 100;
+ }
+ const double bytes = (double) NE * blob;
+ std::printf("bench %-7s 6 experts x 40 layers: gate/up %.0f us, down %.0f us, layer %.0f us -> %.0f GB/s, %.2f Tw/s, 40 layers %.1f ms/token\n",
+ iq::name(type), ms_gu * 1e3, ms_dn * 1e3, (ms_gu + ms_dn) * 1e3,
+ bytes / ((ms_gu + ms_dn) * 1e-3) / 1e9,
+ (double) NE * 3 * FF * E / ((ms_gu + ms_dn) * 1e-3) / 1e12, 40.0 * (ms_gu + ms_dn));
+}
+
+int main(int argc, char** argv) {
+ std::vector<int> types = {16, 17, 22, 18, 21, 23};
+ for (int i = 1; i < argc; ++i) {
+ if (std::string(argv[i]) == "--bench") {
+ std::vector<int> bt;
+ if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
+ else bt = types;
+ for (int t : bt) bench(t);
+ return 0;
+ } else if (std::string(argv[i]) == "--type" && i + 1 < argc) {
+ types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; }
+ } else if (std::string(argv[i]) == "--iq256" && i + 1 < argc) {
+ gg::gg_cpu_set_iq256(std::atoi(argv[++i]));
+ }
+ }
+ std::mt19937_64 rng(11);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ int fails = 0;
+ auto report = [&](const char* what, int type, double worst, double tol) {
+ const bool ok = worst <= tol;
+ std::printf("%-7s %-22s worst %.2e (tol %.0e) %s\n", iq::name(type), what, worst, tol, ok ? "ok" : "FAIL");
+ fails += !ok;
+ };
+ for (int type : types) {
+ if (!iq::supported(type)) { std::printf("%-7s not a device type here\n", iq::name(type)); ++fails; continue; }
+ const int E = 512, FF = 256, NEXP = 3, T = 4;
+ const float lim = 7.0f;
+ const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };
+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
+ std::vector<Mat> G(NEXP), U(NEXP), D(NEXP);
+ auto refh = [&](int e, int r, const float* xv, double& scale) {
+ double sg, su;
+ const double g = dotd(&G[e].w[(size_t) r * E], xv, E, &sg), u = dotd(&U[e].w[(size_t) r * E], xv, E, &su);
+ scale = sg + su + 1e-30;
+ return swiglu(g, u, lim);
+ };
+ std::vector<uint8_t> blobs((size_t) NEXP * blob, 0);
+ for (int e = 0; e < NEXP; ++e) {
+ G[e] = make_mat(rng, type, FF, E); U[e] = make_mat(rng, type, FF, E); D[e] = make_mat(rng, type, E, FF);
+ std::memcpy(&blobs[e * blob + off[0]], G[e].bytes.data(), G[e].bytes.size());
+ std::memcpy(&blobs[e * blob + off[1]], U[e].bytes.data(), U[e].bytes.size());
+ std::memcpy(&blobs[e * blob + off[2]], D[e].bytes.data(), D[e].bytes.size());
+ }
+ uint8_t* dblob = up(blobs);
+ std::vector<float> x((size_t) T * E);
+ for (auto& v : x) v = nd(rng) * (1.0f + 3.0f * (rng() % 7 == 0));
+ std::vector<Q8> qx(T);
+ std::vector<int8_t> xq((size_t) T * E); std::vector<float> xd((size_t) T * E / 32);
+ for (int t = 0; t < T; ++t) { qx[t] = quant(&x[(size_t) t * E], E); std::memcpy(&xq[(size_t) t * E], qx[t].q.data(), E); std::memcpy(&xd[(size_t) t * E / 32], qx[t].d.data(), E / 32 * 4); }
+ const int8_t* dxq = up(xq); const float* dxd = up(xd);
+
+ // ---- token path
+ std::vector<float> href((size_t) NEXP * FF);
+ std::vector<Q8> qh(NEXP);
+ {
+ std::vector<unsigned long long> plan(NEXP);
+ for (int e = 0; e < NEXP; ++e) plan[e] = (unsigned long long) (dblob + e * blob);
+ iq::Args a{};
+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = lim; a.xq = dxq; a.xd = dxd; a.tg = a.td = type;
+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
+ float* dh; CK(hipMalloc(&dh, (size_t) NEXP * FF * 4)); a.h = dh;
+ if (!iq::gate_up(a, NEXP, 1, 0)) { std::printf("gate_up refused\n"); return 1; }
+ CK(hipDeviceSynchronize());
+ const auto h = down_(dh, (size_t) NEXP * FF);
+ double worst = 0;
+ for (int e = 0; e < NEXP; ++e)
+ for (int r = 0; r < FF; ++r) {
+ double sc;
+ const double ref = refh(e, r, qx[0].xhat.data(), sc);
+ href[(size_t) e * FF + r] = (float) ref;
+ worst = std::max(worst, std::fabs(h[(size_t) e * FF + r] - ref) / sc);
+ }
+ report("token gate_up", type, worst, 1e-5);
+ std::vector<int8_t> hq((size_t) NEXP * FF); std::vector<float> hd((size_t) NEXP * FF / 32);
+ for (int e = 0; e < NEXP; ++e) { qh[e] = quant(&href[(size_t) e * FF], FF); std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }
+ a.hq = up(hq); a.hd = up(hd);
+ float* dp; CK(hipMalloc(&dp, (size_t) NEXP * E * 4)); a.part = dp;
+ if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }
+ CK(hipDeviceSynchronize());
+ const auto part = down_(dp, (size_t) NEXP * E);
+ worst = 0;
+ for (int e = 0; e < NEXP; ++e)
+ for (int r = 0; r < E; ++r) {
+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], qh[e].xhat.data(), FF, &sa);
+ worst = std::max(worst, std::fabs(part[(size_t) e * E + r] - ref) / (sa + 1e-30));
+ }
+ report("token down", type, worst, 1e-6);
+
+ // ---- the prompt sets: expert 0 owns rows [0, 37), expert 2 [37, 42)
+ const int NR = 42;
+ std::vector<int> bounds = {0, 37, 37, 42}, idx(NR);
+ for (int i = 0; i < NR; ++i) idx[i] = i % T;
+ iq::SetArgs q{};
+ q.wbase = dblob; q.stride = blob; for (int r = 0; r < 3; ++r) q.tr[r] = off[r];
+ q.bounds = up(bounds); q.row0 = 0; q.max_rows = 37; q.n_exp = 3; q.n_embd = E; q.n_ff = FF; q.limit = lim;
+ q.tg = q.td = type; q.xf = up(x); q.idx = up(idx); q.ldx = E;
+ uint16_t* dh16; CK(hipMalloc(&dh16, (size_t) NR * FF * 2)); q.h = dh16; q.ldo = FF;
+ if (!iq::set_gate_up(q, 0)) { std::printf("set_gate_up refused\n"); return 1; }
+ CK(hipDeviceSynchronize());
+ const auto h16 = down_(dh16, (size_t) NR * FF);
+ double w2 = 0;
+ for (int e = 0; e < 3; ++e)
+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr)
+ for (int r = 0; r < FF; ++r) {
+ double sc;
+ const double ref = refh(e, r, &x[(size_t) idx[lr] * E], sc);
+ w2 = std::max(w2, std::fabs((double) h2f(h16[(size_t) lr * FF + r]) - ref) / sc);
+ }
+ report("set gate_up (f16)", type, w2, 1e-2);
+ q.xh = dh16; q.ldx = FF;
+ float* dy; CK(hipMalloc(&dy, (size_t) NR * E * 4)); q.y = dy; q.ldo = E;
+ if (!iq::set_down(q, 0)) { std::printf("set_down refused\n"); return 1; }
+ CK(hipDeviceSynchronize());
+ const auto y = down_(dy, (size_t) NR * E);
+ w2 = 0;
+ for (int e = 0; e < 3; ++e)
+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {
+ std::vector<float> hf(FF);
+ for (int c = 0; c < FF; ++c) hf[c] = h2f(h16[(size_t) lr * FF + c]);
+ for (int r = 0; r < E; ++r) {
+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hf.data(), FF, &sa);
+ w2 = std::max(w2, std::fabs(y[(size_t) lr * E + r] - ref) / (sa + 1e-30));
+ }
+ }
+ // (the prompt sets' tiles take F16 operands for the i-quants: the tile's own rounding, not the token path's)
+ report("set down", type, w2, 3e-3);
+ }
+
+ // ---- the CPU lane on the same experts and inputs (ggml-cpu's rows, or Strata's AVX2 ones with DS4_GG_IQ256=1)
+ {
+ gg::IqCpu fm; std::string err;
+ if (!gg::iq_cpu_fmt(type, type, E, FF, fm, err)) { std::printf("%-7s cpu lane: %s\n", iq::name(type), err.c_str()); ++fails; }
+ else {
+ std::vector<uint8_t> act(fm.act_bytes), hact(fm.h_bytes);
+ gg::iq_cpu_quant(fm.gu_act, x.data(), E, act.data());
+ // (the CPU lane rounds the activation to its own type - Q8_K, one F32 scale per 256 values - so the
+ // reference is that quantized activation, not the device's per-32 one)
+ const std::vector<float> xhat = act_values(fm.gu_act, act.data(), E);
+ double wg = 0, wd = 0;
+ for (int e = 0; e < NEXP; ++e) {
+ std::vector<float> hc(FF), yc(E);
+ float* hp[1] = {hc.data()}; float* yp[1] = {yc.data()};
+ const void* ap[1] = {act.data()};
+ gg::iq_cpu_gate_up(fm, &blobs[e * blob + off[0]], &blobs[e * blob + off[1]], ap, 1, hp, 0, FF, lim);
+ for (int r = 0; r < FF; ++r) {
+ double sg, su;
+ const double g = dotd(&G[e].w[(size_t) r * E], xhat.data(), E, &sg);
+ const double u = dotd(&U[e].w[(size_t) r * E], xhat.data(), E, &su);
+ wg = std::max(wg, std::fabs(hc[r] - swiglu(g, u, lim)) / (sg + su + 1e-30));
+ }
+ gg::iq_cpu_quant(fm.d_act, &href[(size_t) e * FF], FF, hact.data());
+ const std::vector<float> hhat = act_values(fm.d_act, hact.data(), FF);
+ const void* hpp[1] = {hact.data()};
+ gg::iq_cpu_down(fm, &blobs[e * blob + off[2]], hpp, 1, yp, 0, E);
+ for (int r = 0; r < E; ++r) { double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hhat.data(), FF, &sa); wd = std::max(wd, std::fabs(yc[r] - ref) / (sa + 1e-30)); }
+ }
+ report("cpu gate_up vs ref", type, wg, 1e-5);
+ report("cpu down vs ref", type, wd, 1e-5);
+ std::printf("%-7s cpu rows: %s\n", iq::name(type), gg::iq_cpu_own_rows(type) ? "Strata AVX2" : "ggml-cpu");
+ }
+ }
+ CK(hipFree(dblob));
+ }
+ std::printf("%s\n", fails ? "FAILED" : "ALL OK");
+ return fails;
+}