Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 cpu lane: exact AVX-512 rows (iq512x_rows, 1-6 tokens and prompt tiles), opt-in DS4_GG_IQ512=1 until measured on Zen 4

Niko122149e35e8bb735
diff --git a/include/strata/kernels/cpu/iq_avx512.hpp b/include/strata/kernels/cpu/iq_avx512.hpp
index a9a71c461..ae21dfd44 100644
--- a/include/strata/kernels/cpu/iq_avx512.hpp
+++ b/include/strata/kernels/cpu/iq_avx512.hpp
@@ -15,4 +15,11 @@ void iq512_gu_rows(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_
void iq512_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
float* const* out, int r0, int r1);
+/// The DeepSeek-V4.1 CPU lane's rows (src/dsv41/ds4_gg_cpu.cpp): the same 64-value decode, summed in ggml's order, so a token's
+/// dot is ggml-cpu's AVX2 vec_dot bit for bit at any number of tokens (1-6 in one pass, 7 and more as a prompt chunk's tiles).
+/// The caller checks cpu_avx512_ok() (AVX-512 F/BW/VL/DQ/VNNI/VBMI).
+bool iq512x_supported(int ggml_type) noexcept;
+void iq512x_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
+ float* const* out, int r0, int r1);
+
} // namespace strata::kernels::cpu
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cpp
index 5ec1e6763..dfab6765a 100644
--- a/src/dsv41/ds4_gg_cpu.cpp
+++ b/src/dsv41/ds4_gg_cpu.cpp
@@ -12,7 +12,9 @@
#if defined(STRATA_DSV41_TQ_CPU)
#include "ggml-cpu.h"
#include "ggml.h"
+#include "strata/kernels/cpu/expert_layout.hpp"
#include "strata/kernels/cpu/iq_avx2.hpp"
+#include "strata/kernels/cpu/iq_avx512.hpp"
#include <mutex>
#include <vector>
@@ -86,12 +88,12 @@ void gg_cpu_quant(int act, const float* x, int k, void* dst) {
static int g_iq256 = -1; // -1: from the environment and the CPU, 0 off, 1 always, 2 from min_tokens(type)
void gg_cpu_set_iq256(int mode) { g_iq256 = mode; }
static int min_tokens(int type) { return type == 21 ? 1 : type == 29 ? 3 : 2; }
-static bool own_ok(int type) {
- static std::once_flag once[32];
- static bool ok[32];
+static bool own_ok(int type, int impl) {
+ static std::once_flag once[2][32];
+ static bool ok[2][32];
const int k = type & 31;
- std::call_once(once[k], [&] {
- ok[k] = false;
+ std::call_once(once[impl][k], [&] {
+ ok[impl][k] = false;
const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);
const ggml_type_traits_cpu* q8 = tc ? ggml_get_type_traits_cpu(tc->vec_dot_type) : nullptr;
if (tc == nullptr || tc->vec_dot == nullptr || q8 == nullptr || q8->from_float == nullptr) return;
@@ -125,33 +127,57 @@ static bool own_ok(int type) {
p1[(size_t) t] = o1[(size_t) t].data();
for (int r = 0; r < rows; ++r) tc->vec_dot(n, &o0[(size_t) t][(size_t) r], 0, w.data() + (size_t) r * rb, 0, ap[(size_t) t], 0, 1);
}
- strata::kernels::cpu::iq256x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);
+ if (impl == 1) strata::kernels::cpu::iq512x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);
+ else strata::kernels::cpu::iq256x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);
for (int t = 0; t < nt; ++t)
if (std::memcmp(o0[(size_t) t].data(), o1[(size_t) t].data(), rows * sizeof(float)) != 0) {
if (dbg() || std::getenv("DS4_GG_VERBOSE")) std::fprintf(stderr, "ds4: CPU lane type %d: AVX2 rows differ from ggml-cpu's dot at %d tokens, not used\n", type, nt);
return;
}
}
- ok[k] = true;
+ ok[impl][k] = true;
});
- return ok[k];
+ return ok[impl][k];
}
-static bool use_iq256(int type, int nt) {
+// DS4_GG_IQ512: 1 takes the AVX-512 rows (bit-identical too) where the CPU has AVX-512 F/BW/VL/DQ/VNNI/VBMI, 0 never; unset: the rule
+// below, from the measurements on Zen 4.
+static int g_iq512 = -1;
+static bool want_512(int type, int nt) {
+ if (g_iq512 < 0) {
+ const char* v = std::getenv("DS4_GG_IQ512");
+ g_iq512 = !strata::kernels::cpu::cpu_avx512_ok() ? 0 : v == nullptr ? 2 : std::atoi(v) != 0 ? 1 : 0;
+ }
+ if (g_iq512 == 0 || !strata::kernels::cpu::iq512x_supported(type)) return false;
+ if (g_iq512 == 1) return true;
+ (void) nt;
+ return false; // (the default rule is filled in from the Zen 4 numbers)
+}
+// 0: ggml-cpu's dot, 1: the AVX2 rows, 2: the AVX-512 rows
+static int use_iq256(int type, int nt) {
if (g_iq256 < 0) {
const char* v = std::getenv("DS4_GG_IQ256");
// (ggml-cpu's own answer: built with AVX2 + FMA + F16C, which is also what makes its dots the AVX2 ones these rows match)
const bool avx2 = ggml_cpu_has_avx2() && ggml_cpu_has_fma() && ggml_cpu_has_f16c();
g_iq256 = !avx2 ? 0 : v == nullptr ? 2 : std::atoi(v) != 0 ? 1 : 0;
}
- if (g_iq256 == 0 || !strata::kernels::cpu::iq256x_supported(type)) return false;
- if (g_iq256 == 2 && nt < min_tokens(type)) return false;
- return own_ok(type);
+ if (g_iq256 == 0) return 0;
+ if (want_512(type, nt) && own_ok(type, 1)) return 2;
+ if (!strata::kernels::cpu::iq256x_supported(type)) return 0;
+ if (g_iq256 == 2 && nt < min_tokens(type)) return 0;
+ return own_ok(type, 0) ? 1 : 0;
}
static void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
- if (nt > 0 && use_iq256(type, nt)) {
- strata::kernels::cpu::iq256x_rows(type, w, rb, n, act, nt, out, r0, r1);
- return;
+ if (nt > 0) {
+ const int impl = use_iq256(type, nt);
+ if (impl == 2) {
+ strata::kernels::cpu::iq512x_rows(type, w, rb, n, act, nt, out, r0, r1);
+ return;
+ }
+ if (impl == 1) {
+ strata::kernels::cpu::iq256x_rows(type, w, rb, n, act, nt, out, r0, r1);
+ return;
+ }
}
const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);
if (dbg()) std::fprintf(stderr, "gg dots type %d n %d rows %d..%d nt %d\n", type, n, r0, r1, nt);
diff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cpp
index e21a4d431..b7c6e56ff 100644
--- a/src/kernels/cpu/iq_avx512.cpp
+++ b/src/kernels/cpu/iq_avx512.cpp
@@ -19,6 +19,7 @@
#include <cmath>
#include <cstdlib>
#include <cstring>
+#include <vector>
namespace strata::kernels::cpu {
namespace {
@@ -240,6 +241,138 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*
}
}
+
+// ---- the DeepSeek CPU lane's exact rows (iq512x_rows): the same 64-value decode, but summed as ggml's AVX2 dots sum, so a
+// token's row is ggml-cpu's vec_dot bit for bit at any number of tokens: the integer sums of a block fold to ggml's eight lanes
+// (low half = the even 32-value chunk, high half = the odd one), the block scale is f16(d) * y.d, eight float lanes accumulate
+// across blocks, and K * ggml's hsum comes last. vpdpwssd is the madd and the add in one.
+inline __m256i fold256(__m512i a) { return _mm256_add_epi32(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64(a, 1)); }
+
+inline float hsum8_ggml(__m256 x) {
+ __m128 s = _mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x));
+ s = _mm_add_ps(s, _mm_movehl_ps(s, s));
+ s = _mm_add_ss(s, _mm_movehdup_ps(s));
+ return _mm_cvtss_f32(s);
+}
+
+template <int TY, int NT>
+inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {
+ __m256 accf[NT];
+ for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
+ const __m512i zero = _mm512_setzero_si512();
+ for (int i = 0; i < nblocks; ++i) {
+ const uint8_t* blk = row + (size_t) i * Fmt<TY>::bytes;
+ if (prefetch_ahead > 0) {
+ const uint8_t* ahead = blk + (size_t) prefetch_ahead;
+ _mm_prefetch((const char*) ahead, _MM_HINT_T0);
+ _mm_prefetch((const char*) ahead + 64, _MM_HINT_T0);
+ }
+ __m512i acci[NT];
+ for (int t = 0; t < NT; ++t) acci[t] = _mm512_setzero_si512();
+ for (int j = 0; j < 4; ++j) {
+ __m512i g, sc;
+ __mmask64 m;
+ Fmt<TY>::decode(blk, j, g, m, sc);
+ for (int t = 0; t < NT; ++t) {
+ const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j));
+ const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);
+ acci[t] = _mm512_dpwssd_epi32(acci[t], _mm512_maddubs_epi16(g, ys), sc);
+ }
+ }
+ const float dx = h2f(u16(blk));
+ for (int t = 0; t < NT; ++t)
+ accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold256(acci[t])), accf[t]);
+ }
+ for (int t = 0; t < NT; ++t) res[t] = Fmt<TY>::K * hsum8_ggml(accf[t]);
+}
+
+template <int TY, int NT>
+void dot_rows_x(const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0, int r1) {
+ const block_q8_K* y[NT];
+ for (int t = 0; t < NT; ++t) y[t] = (const block_q8_K*) act[t];
+ float res[NT];
+ for (int r = r0; r < r1; ++r) {
+ row_dot_x<TY, NT>(w + (size_t) r * row_bytes, n / QK_K, y, res);
+ for (int t = 0; t < NT; ++t) out[t][r] = res[t];
+ }
+}
+
+// many tokens an expert (the prompt): signed int8 weights decoded once per block, activations as u8 (q8 + 128, once a call),
+// sum(w * q) = sum(w * (q + 128)) - 128 * sum(w) with the second sum once a weight block; tokens in tiles of 16 and rows in
+// groups of 8 so the tile's slice of a block and the group's accumulators stay in L1 (the AVX2 form's layout)
+template <int TY>
+void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int T, float* const* out, int r0, int r1) {
+ constexpr int kTile = 16, kRows = 8;
+ const int nb = n / QK_K;
+ static thread_local std::vector<uint8_t> ubuf;
+ static thread_local std::vector<float> accb;
+ ubuf.resize((size_t) (T < kTile ? T : kTile) * n);
+ accb.resize((size_t) kRows * kTile * 8);
+ const __m512i zero = _mm512_setzero_si512(), one8 = _mm512_set1_epi8(1), flip = _mm512_set1_epi8((char) 0x80);
+ for (int t0 = 0; t0 < T; t0 += kTile) {
+ const int tt = T - t0 < kTile ? T - t0 : kTile;
+ const block_q8_K* y[kTile];
+ for (int t = 0; t < tt; ++t) {
+ y[t] = (const block_q8_K*) act[t0 + t];
+ uint8_t* u = ubuf.data() + (size_t) t * n;
+ for (int i = 0; i < nb; ++i)
+ for (int j = 0; j < 4; ++j)
+ _mm512_storeu_si512((void*) (u + (size_t) i * 256 + 64 * j),
+ _mm512_xor_si512(_mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j)), flip));
+ }
+ for (int rg = r0; rg < r1; rg += kRows) {
+ const int rr = r1 - rg < kRows ? r1 - rg : kRows;
+ for (int k = 0; k < rr * tt; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());
+ for (int i = 0; i < nb; ++i)
+ for (int q = 0; q < rr; ++q) {
+ const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * Fmt<TY>::bytes;
+ if (prefetch_ahead > 0) {
+ _mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);
+ _mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);
+ }
+ __m512i w8[4], sc[4];
+ __m512i corr = zero;
+ for (int j = 0; j < 4; ++j) {
+ __m512i g, s;
+ __mmask64 m;
+ Fmt<TY>::decode(blk, j, g, m, s);
+ w8[j] = _mm512_mask_sub_epi8(g, m, zero, g);
+ sc[j] = s;
+ corr = _mm512_dpwssd_epi32(corr, _mm512_maddubs_epi16(one8, w8[j]), s);
+ }
+ const __m256i corr128 = _mm256_slli_epi32(fold256(corr), 7);
+ const float dx = h2f(u16(blk));
+ float* accq = &accb[(size_t) q * tt * 8];
+ for (int t = 0; t < tt; ++t) {
+ const uint8_t* u = ubuf.data() + (size_t) t * n + (size_t) i * 256;
+ __m512i a = zero;
+ for (int j = 0; j < 4; ++j)
+ a = _mm512_dpwssd_epi32(a, _mm512_maddubs_epi16(_mm512_loadu_si512((const void*) (u + 64 * j)), w8[j]), sc[j]);
+ const __m256i s = _mm256_sub_epi32(fold256(a), corr128);
+ float* ap = accq + (size_t) t * 8;
+ _mm256_storeu_ps(ap, _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(s), _mm256_loadu_ps(ap)));
+ }
+ }
+ for (int q = 0; q < rr; ++q)
+ for (int t = 0; t < tt; ++t)
+ out[t0 + t][rg + q] = Fmt<TY>::K * hsum8_ggml(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));
+ }
+ }
+}
+
+template <int TY>
+void dot_rows_x_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0, int r1) {
+ switch (nt) {
+ case 1: dot_rows_x<TY, 1>(w, row_bytes, n, act, out, r0, r1); break;
+ case 2: dot_rows_x<TY, 2>(w, row_bytes, n, act, out, r0, r1); break;
+ case 3: dot_rows_x<TY, 3>(w, row_bytes, n, act, out, r0, r1); break;
+ case 4: dot_rows_x<TY, 4>(w, row_bytes, n, act, out, r0, r1); break;
+ case 5: dot_rows_x<TY, 5>(w, row_bytes, n, act, out, r0, r1); break;
+ case 6: dot_rows_x<TY, 6>(w, row_bytes, n, act, out, r0, r1); break;
+ default: dot_rows_wide<TY>(w, row_bytes, n, act, nt, out, r0, r1); break;
+ }
+}
+
} // namespace
void iq512_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,
@@ -266,4 +399,18 @@ void iq512_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*
}
}
+bool iq512x_supported(int type) noexcept { return type == 16 || type == 17 || type == 18 || type == 21 || type == 22; }
+
+void iq512x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out,
+ int r0, int r1) {
+ switch (type) {
+ case 16: dot_rows_x_nt<16>(nt, w, row_bytes, n, act, out, r0, r1); break;
+ case 17: dot_rows_x_nt<17>(nt, w, row_bytes, n, act, out, r0, r1); break;
+ case 18: dot_rows_x_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;
+ case 21: dot_rows_x_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;
+ case 22: dot_rows_x_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;
+ default: break;
+ }
+}
+
} // namespace strata::kernels::cpu