Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 cpu lane: exact AVX-512 rows (iq512x_rows, 1-6 tokens and prompt tiles), opt-in DS4_GG_IQ512=1 until measured on Zen 4
Niko122149e35e8bb735
diff --git a/include/strata/kernels/cpu/iq_avx512.hpp b/include/strata/kernels/cpu/iq_avx512.hppindex a9a71c461..ae21dfd44 100644--- a/include/strata/kernels/cpu/iq_avx512.hpp+++ b/include/strata/kernels/cpu/iq_avx512.hpp@@ -15,4 +15,11 @@ void iq512_gu_rows(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_void iq512_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,float* const* out, int r0, int r1);+/// The DeepSeek-V4.1 CPU lane's rows (src/dsv41/ds4_gg_cpu.cpp): the same 64-value decode, summed in ggml's order, so a token's+/// dot is ggml-cpu's AVX2 vec_dot bit for bit at any number of tokens (1-6 in one pass, 7 and more as a prompt chunk's tiles).+/// The caller checks cpu_avx512_ok() (AVX-512 F/BW/VL/DQ/VNNI/VBMI).+bool iq512x_supported(int ggml_type) noexcept;+void iq512x_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,+ float* const* out, int r0, int r1);+} // namespace strata::kernels::cpudiff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cppindex 5ec1e6763..dfab6765a 100644--- a/src/dsv41/ds4_gg_cpu.cpp+++ b/src/dsv41/ds4_gg_cpu.cpp@@ -12,7 +12,9 @@#if defined(STRATA_DSV41_TQ_CPU)#include "ggml-cpu.h"#include "ggml.h"+#include "strata/kernels/cpu/expert_layout.hpp"#include "strata/kernels/cpu/iq_avx2.hpp"+#include "strata/kernels/cpu/iq_avx512.hpp"#include <mutex>#include <vector>@@ -86,12 +88,12 @@ void gg_cpu_quant(int act, const float* x, int k, void* dst) {static int g_iq256 = -1; // -1: from the environment and the CPU, 0 off, 1 always, 2 from min_tokens(type)void gg_cpu_set_iq256(int mode) { g_iq256 = mode; }static int min_tokens(int type) { return type == 21 ? 1 : type == 29 ? 3 : 2; }-static bool own_ok(int type) {- static std::once_flag once[32];- static bool ok[32];+static bool own_ok(int type, int impl) {+ static std::once_flag once[2][32];+ static bool ok[2][32];const int k = type & 31;- std::call_once(once[k], [&] {- ok[k] = false;+ std::call_once(once[impl][k], [&] {+ ok[impl][k] = false;const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);const ggml_type_traits_cpu* q8 = tc ? ggml_get_type_traits_cpu(tc->vec_dot_type) : nullptr;if (tc == nullptr || tc->vec_dot == nullptr || q8 == nullptr || q8->from_float == nullptr) return;@@ -125,33 +127,57 @@ static bool own_ok(int type) {p1[(size_t) t] = o1[(size_t) t].data();for (int r = 0; r < rows; ++r) tc->vec_dot(n, &o0[(size_t) t][(size_t) r], 0, w.data() + (size_t) r * rb, 0, ap[(size_t) t], 0, 1);}- strata::kernels::cpu::iq256x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);+ if (impl == 1) strata::kernels::cpu::iq512x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);+ else strata::kernels::cpu::iq256x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);for (int t = 0; t < nt; ++t)if (std::memcmp(o0[(size_t) t].data(), o1[(size_t) t].data(), rows * sizeof(float)) != 0) {if (dbg() || std::getenv("DS4_GG_VERBOSE")) std::fprintf(stderr, "ds4: CPU lane type %d: AVX2 rows differ from ggml-cpu's dot at %d tokens, not used\n", type, nt);return;}}- ok[k] = true;+ ok[impl][k] = true;});- return ok[k];+ return ok[impl][k];}-static bool use_iq256(int type, int nt) {+// DS4_GG_IQ512: 1 takes the AVX-512 rows (bit-identical too) where the CPU has AVX-512 F/BW/VL/DQ/VNNI/VBMI, 0 never; unset: the rule+// below, from the measurements on Zen 4.+static int g_iq512 = -1;+static bool want_512(int type, int nt) {+ if (g_iq512 < 0) {+ const char* v = std::getenv("DS4_GG_IQ512");+ g_iq512 = !strata::kernels::cpu::cpu_avx512_ok() ? 0 : v == nullptr ? 2 : std::atoi(v) != 0 ? 1 : 0;+ }+ if (g_iq512 == 0 || !strata::kernels::cpu::iq512x_supported(type)) return false;+ if (g_iq512 == 1) return true;+ (void) nt;+ return false; // (the default rule is filled in from the Zen 4 numbers)+}+// 0: ggml-cpu's dot, 1: the AVX2 rows, 2: the AVX-512 rows+static int use_iq256(int type, int nt) {if (g_iq256 < 0) {const char* v = std::getenv("DS4_GG_IQ256");// (ggml-cpu's own answer: built with AVX2 + FMA + F16C, which is also what makes its dots the AVX2 ones these rows match)const bool avx2 = ggml_cpu_has_avx2() && ggml_cpu_has_fma() && ggml_cpu_has_f16c();g_iq256 = !avx2 ? 0 : v == nullptr ? 2 : std::atoi(v) != 0 ? 1 : 0;}- if (g_iq256 == 0 || !strata::kernels::cpu::iq256x_supported(type)) return false;- if (g_iq256 == 2 && nt < min_tokens(type)) return false;- return own_ok(type);+ if (g_iq256 == 0) return 0;+ if (want_512(type, nt) && own_ok(type, 1)) return 2;+ if (!strata::kernels::cpu::iq256x_supported(type)) return 0;+ if (g_iq256 == 2 && nt < min_tokens(type)) return 0;+ return own_ok(type, 0) ? 1 : 0;}static void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {- if (nt > 0 && use_iq256(type, nt)) {- strata::kernels::cpu::iq256x_rows(type, w, rb, n, act, nt, out, r0, r1);- return;+ if (nt > 0) {+ const int impl = use_iq256(type, nt);+ if (impl == 2) {+ strata::kernels::cpu::iq512x_rows(type, w, rb, n, act, nt, out, r0, r1);+ return;+ }+ if (impl == 1) {+ strata::kernels::cpu::iq256x_rows(type, w, rb, n, act, nt, out, r0, r1);+ return;+ }}const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);if (dbg()) std::fprintf(stderr, "gg dots type %d n %d rows %d..%d nt %d\n", type, n, r0, r1, nt);diff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cppindex e21a4d431..b7c6e56ff 100644--- a/src/kernels/cpu/iq_avx512.cpp+++ b/src/kernels/cpu/iq_avx512.cpp@@ -19,6 +19,7 @@#include <cmath>#include <cstdlib>#include <cstring>+#include <vector>namespace strata::kernels::cpu {namespace {@@ -240,6 +241,138 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*}}++// ---- the DeepSeek CPU lane's exact rows (iq512x_rows): the same 64-value decode, but summed as ggml's AVX2 dots sum, so a+// token's row is ggml-cpu's vec_dot bit for bit at any number of tokens: the integer sums of a block fold to ggml's eight lanes+// (low half = the even 32-value chunk, high half = the odd one), the block scale is f16(d) * y.d, eight float lanes accumulate+// across blocks, and K * ggml's hsum comes last. vpdpwssd is the madd and the add in one.+inline __m256i fold256(__m512i a) { return _mm256_add_epi32(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64(a, 1)); }++inline float hsum8_ggml(__m256 x) {+ __m128 s = _mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x));+ s = _mm_add_ps(s, _mm_movehl_ps(s, s));+ s = _mm_add_ss(s, _mm_movehdup_ps(s));+ return _mm_cvtss_f32(s);+}++template <int TY, int NT>+inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {+ __m256 accf[NT];+ for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();+ const __m512i zero = _mm512_setzero_si512();+ for (int i = 0; i < nblocks; ++i) {+ const uint8_t* blk = row + (size_t) i * Fmt<TY>::bytes;+ if (prefetch_ahead > 0) {+ const uint8_t* ahead = blk + (size_t) prefetch_ahead;+ _mm_prefetch((const char*) ahead, _MM_HINT_T0);+ _mm_prefetch((const char*) ahead + 64, _MM_HINT_T0);+ }+ __m512i acci[NT];+ for (int t = 0; t < NT; ++t) acci[t] = _mm512_setzero_si512();+ for (int j = 0; j < 4; ++j) {+ __m512i g, sc;+ __mmask64 m;+ Fmt<TY>::decode(blk, j, g, m, sc);+ for (int t = 0; t < NT; ++t) {+ const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j));+ const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);+ acci[t] = _mm512_dpwssd_epi32(acci[t], _mm512_maddubs_epi16(g, ys), sc);+ }+ }+ const float dx = h2f(u16(blk));+ for (int t = 0; t < NT; ++t)+ accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold256(acci[t])), accf[t]);+ }+ for (int t = 0; t < NT; ++t) res[t] = Fmt<TY>::K * hsum8_ggml(accf[t]);+}++template <int TY, int NT>+void dot_rows_x(const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0, int r1) {+ const block_q8_K* y[NT];+ for (int t = 0; t < NT; ++t) y[t] = (const block_q8_K*) act[t];+ float res[NT];+ for (int r = r0; r < r1; ++r) {+ row_dot_x<TY, NT>(w + (size_t) r * row_bytes, n / QK_K, y, res);+ for (int t = 0; t < NT; ++t) out[t][r] = res[t];+ }+}++// many tokens an expert (the prompt): signed int8 weights decoded once per block, activations as u8 (q8 + 128, once a call),+// sum(w * q) = sum(w * (q + 128)) - 128 * sum(w) with the second sum once a weight block; tokens in tiles of 16 and rows in+// groups of 8 so the tile's slice of a block and the group's accumulators stay in L1 (the AVX2 form's layout)+template <int TY>+void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int T, float* const* out, int r0, int r1) {+ constexpr int kTile = 16, kRows = 8;+ const int nb = n / QK_K;+ static thread_local std::vector<uint8_t> ubuf;+ static thread_local std::vector<float> accb;+ ubuf.resize((size_t) (T < kTile ? T : kTile) * n);+ accb.resize((size_t) kRows * kTile * 8);+ const __m512i zero = _mm512_setzero_si512(), one8 = _mm512_set1_epi8(1), flip = _mm512_set1_epi8((char) 0x80);+ for (int t0 = 0; t0 < T; t0 += kTile) {+ const int tt = T - t0 < kTile ? T - t0 : kTile;+ const block_q8_K* y[kTile];+ for (int t = 0; t < tt; ++t) {+ y[t] = (const block_q8_K*) act[t0 + t];+ uint8_t* u = ubuf.data() + (size_t) t * n;+ for (int i = 0; i < nb; ++i)+ for (int j = 0; j < 4; ++j)+ _mm512_storeu_si512((void*) (u + (size_t) i * 256 + 64 * j),+ _mm512_xor_si512(_mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j)), flip));+ }+ for (int rg = r0; rg < r1; rg += kRows) {+ const int rr = r1 - rg < kRows ? r1 - rg : kRows;+ for (int k = 0; k < rr * tt; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());+ for (int i = 0; i < nb; ++i)+ for (int q = 0; q < rr; ++q) {+ const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * Fmt<TY>::bytes;+ if (prefetch_ahead > 0) {+ _mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);+ _mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);+ }+ __m512i w8[4], sc[4];+ __m512i corr = zero;+ for (int j = 0; j < 4; ++j) {+ __m512i g, s;+ __mmask64 m;+ Fmt<TY>::decode(blk, j, g, m, s);+ w8[j] = _mm512_mask_sub_epi8(g, m, zero, g);+ sc[j] = s;+ corr = _mm512_dpwssd_epi32(corr, _mm512_maddubs_epi16(one8, w8[j]), s);+ }+ const __m256i corr128 = _mm256_slli_epi32(fold256(corr), 7);+ const float dx = h2f(u16(blk));+ float* accq = &accb[(size_t) q * tt * 8];+ for (int t = 0; t < tt; ++t) {+ const uint8_t* u = ubuf.data() + (size_t) t * n + (size_t) i * 256;+ __m512i a = zero;+ for (int j = 0; j < 4; ++j)+ a = _mm512_dpwssd_epi32(a, _mm512_maddubs_epi16(_mm512_loadu_si512((const void*) (u + 64 * j)), w8[j]), sc[j]);+ const __m256i s = _mm256_sub_epi32(fold256(a), corr128);+ float* ap = accq + (size_t) t * 8;+ _mm256_storeu_ps(ap, _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(s), _mm256_loadu_ps(ap)));+ }+ }+ for (int q = 0; q < rr; ++q)+ for (int t = 0; t < tt; ++t)+ out[t0 + t][rg + q] = Fmt<TY>::K * hsum8_ggml(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));+ }+ }+}++template <int TY>+void dot_rows_x_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0, int r1) {+ switch (nt) {+ case 1: dot_rows_x<TY, 1>(w, row_bytes, n, act, out, r0, r1); break;+ case 2: dot_rows_x<TY, 2>(w, row_bytes, n, act, out, r0, r1); break;+ case 3: dot_rows_x<TY, 3>(w, row_bytes, n, act, out, r0, r1); break;+ case 4: dot_rows_x<TY, 4>(w, row_bytes, n, act, out, r0, r1); break;+ case 5: dot_rows_x<TY, 5>(w, row_bytes, n, act, out, r0, r1); break;+ case 6: dot_rows_x<TY, 6>(w, row_bytes, n, act, out, r0, r1); break;+ default: dot_rows_wide<TY>(w, row_bytes, n, act, nt, out, r0, r1); break;+ }+}+} // namespacevoid iq512_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,@@ -266,4 +399,18 @@ void iq512_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*}}+bool iq512x_supported(int type) noexcept { return type == 16 || type == 17 || type == 18 || type == 21 || type == 22; }++void iq512x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out,+ int r0, int r1) {+ switch (type) {+ case 16: dot_rows_x_nt<16>(nt, w, row_bytes, n, act, out, r0, r1); break;+ case 17: dot_rows_x_nt<17>(nt, w, row_bytes, n, act, out, r0, r1); break;+ case 18: dot_rows_x_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;+ case 21: dot_rows_x_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;+ case 22: dot_rows_x_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;+ default: break;+ }+}+} // namespace strata::kernels::cpu