Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 cpu lane: one-token AVX-512 rows - sign masks by popcount (IQ2_XXS, IQ2_XS, IQ3_XXS), IQ1_M AVX-512 rows with pdep indices; chosen per CPU (Zen 5 from 1 token)
Niko12211747c8e53882
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 80ba5890c..8a7894466 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1663,7 +1663,7 @@ if(STRATA_NATIVE_EXPERTS)set_source_files_properties(src/kernels/cpu/iq_avx1.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX")else()set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp PROPERTIES COMPILE_OPTIONS- "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512vnni;-mavx512vbmi;-mf16c;-mfma")+ "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512vnni;-mavx512vbmi;-mavx512vpopcntdq;-mbmi2;-mf16c;-mfma")set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cppindex b8da9626f..a54a22902 100644--- a/src/dsv41/ds4_gg_cpu.cpp+++ b/src/dsv41/ds4_gg_cpu.cpp@@ -183,15 +183,16 @@ static bool want_512(int type, int nt) {}if (g_iq512 == 0 || !strata::kernels::cpu::iq512x_supported(type)) return false;if (g_iq512 == 1) return true;- // by CPU (measured, ds4_gg_bench --expert at 16 tokens; the AVX-512 rows tie the AVX2 ones at 4 tokens and lose at one):- // Zen 5 (AMD family 0x1A, Strix Halo): IQ2_XXS 8.1 -> 10.9 GB/s, IQ3_XXS 10.8 -> 13.9;- // Zen 4 (family 0x19, Ryzen 5 7600): IQ2_XXS 3.2 -> 4.0, IQ3_XXS no gain;- // anything else with AVX-512 (Intel): not measured, so the AVX2 rows.- // and only for a prompt chunk's tokens (7 and more)- if (nt < 7) return false;+ // by CPU (measured, ds4_gg_bench --expert; the rows are bit-identical to ggml's dot everywhere):+ // Zen 5 (AMD family 0x1A, Strix Halo, 16 threads, GB/s of weights ggml -> AVX-512 rows): one token IQ2_XXS 35 -> 44, IQ2_XS 37 -> 48,+ // IQ1_M 32 -> 36 (the sign masks by popcount, IQ1_M's indices by pdep: no table loads), IQ3_XXS 43 -> 45; 4 tokens x2.5-3.5; 16 tokens x4.4-4.8;+ // Zen 4 (family 0x19): see the Zen 4 table; anything else with AVX-512 (Intel): not measured, so the AVX2 rowsconst int fam = amd_family();- if (fam >= 0x1A) return type == 16 || type == 18;- if (fam == 0x19) return type == 16;+ if (fam >= 0x1A) {+ if (type == 16 || type == 17 || type == 29) return true;+ return type == 18 && nt >= 2;+ }+ if (fam == 0x19) return nt >= 7 && type == 16;return false;}// 0: ggml-cpu's dot, 1: the AVX2 rows, 2: the AVX-512 rowsdiff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cppindex b7c6e56ff..14330f6ec 100644--- a/src/kernels/cpu/iq_avx512.cpp+++ b/src/kernels/cpu/iq_avx512.cpp@@ -15,6 +15,11 @@#include "ggml-common.h"#include <immintrin.h>+#if defined(_MSC_VER)+#include <intrin.h>+#else+#include <cpuid.h>+#endif#include <cmath>#include <cstdlib>@@ -242,6 +247,61 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*}+// The sign masks of IQ2_XXS / IQ2_XS / IQ3_XXS come from a 7-bit index whose 8th bit is the parity of the other seven (ksigns_iq2xs[i] =+// i | parity(i) << 7). The table needs a load and a few shifts and ors a group of eight values; popcount does it for all eight groups+// of 64 values in one vector: measured on Zen 5, one-token IQ2_XXS rows 33 -> 45-50 GB/s (16 threads, DRAM-sized), cache-resident 16.8 -> 13.4 ns a block.+// (AVX512_VPOPCNTDQ + VL: Zen 4, Zen 5, Ice Lake and later; iq512x_supported() asks for it.)+inline __mmask64 sign_mask_from_idx(__m256i si) {+ const __m256i par = _mm256_slli_epi32(_mm256_and_si256(_mm256_popcnt_epi32(si), _mm256_set1_epi32(1)), 7);+ return _cvtu64_mask64((uint64_t) _mm_cvtsi128_si64(_mm256_cvtepi32_epi8(_mm256_or_si256(si, par))));+}+inline __m256i sign_idx_xxs(uint32_t lo4, uint32_t hi4) { // four 7-bit fields of each of two words -> eight dwords+ const __m256i w = _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_set1_epi32((int) lo4)), _mm_set1_epi32((int) hi4), 1);+ return _mm256_and_si256(_mm256_srlv_epi32(w, _mm256_setr_epi32(0, 7, 14, 21, 0, 7, 14, 21)), _mm256_set1_epi32(127));+}++template <int TY> struct FmtX : Fmt<TY> {};+template <> struct FmtX<16> : Fmt<16> {+ static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {+ const uint8_t* q = b + 2 + 16 * j;+ const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);+ g = _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],+ (long long) iq2xxs_grid[(b0 >> 8) & 255], (long long) iq2xxs_grid[b0 & 255],+ (long long) iq2xxs_grid[a0 >> 24], (long long) iq2xxs_grid[(a0 >> 16) & 255],+ (long long) iq2xxs_grid[(a0 >> 8) & 255], (long long) iq2xxs_grid[a0 & 255]);+ m = sign_mask_from_idx(sign_idx_xxs(a1, b1));+ const int sa = 2 * (int) (a1 >> 28) + 1, sb = 2 * (int) (b1 >> 28) + 1;+ sc = scales4(sa, sa, sb, sb);+ }+};+template <> struct FmtX<17> : Fmt<17> {+ static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {+ const uint8_t* q = b + 2 + 16 * j;+ uint16_t v[8];+ std::memcpy(v, q, 16);+ g = _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],+ (long long) iq2xs_grid[v[5] & 511], (long long) iq2xs_grid[v[4] & 511],+ (long long) iq2xs_grid[v[3] & 511], (long long) iq2xs_grid[v[2] & 511],+ (long long) iq2xs_grid[v[1] & 511], (long long) iq2xs_grid[v[0] & 511]);+ m = sign_mask_from_idx(_mm256_srli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*) q)), 9));+ const uint8_t s0 = b[66 + 2 * j], s1 = b[66 + 2 * j + 1];+ sc = scales4(2 * (s0 & 15) + 1, 2 * (s0 >> 4) + 1, 2 * (s1 & 15) + 1, 2 * (s1 >> 4) + 1);+ }+};+template <> struct FmtX<18> : Fmt<18> {+ static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {+ const uint8_t* q = b + 2 + 16 * j;+ g = _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],+ (int) iq3xxs_grid[q[11]], (int) iq3xxs_grid[q[10]], (int) iq3xxs_grid[q[9]], (int) iq3xxs_grid[q[8]],+ (int) iq3xxs_grid[q[7]], (int) iq3xxs_grid[q[6]], (int) iq3xxs_grid[q[5]], (int) iq3xxs_grid[q[4]],+ (int) iq3xxs_grid[q[3]], (int) iq3xxs_grid[q[2]], (int) iq3xxs_grid[q[1]], (int) iq3xxs_grid[q[0]]);+ const uint32_t a = u32(b + 2 + 64 + 8 * j), c = u32(b + 2 + 64 + 8 * j + 4);+ m = sign_mask_from_idx(sign_idx_xxs(a, c));+ const int sa = 2 * (int) (a >> 28) + 1, sb = 2 * (int) (c >> 28) + 1;+ sc = scales4(sa, sa, sb, sb);+ }+};+// ---- the DeepSeek CPU lane's exact rows (iq512x_rows): the same 64-value decode, but summed as ggml's AVX2 dots sum, so a// token's row is ggml-cpu's vec_dot bit for bit at any number of tokens: the integer sums of a block fold to ggml's eight lanes// (low half = the even 32-value chunk, high half = the odd one), the block scale is f16(d) * y.d, eight float lanes accumulate@@ -261,7 +321,7 @@ inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const*for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();const __m512i zero = _mm512_setzero_si512();for (int i = 0; i < nblocks; ++i) {- const uint8_t* blk = row + (size_t) i * Fmt<TY>::bytes;+ const uint8_t* blk = row + (size_t) i * FmtX<TY>::bytes;if (prefetch_ahead > 0) {const uint8_t* ahead = blk + (size_t) prefetch_ahead;_mm_prefetch((const char*) ahead, _MM_HINT_T0);@@ -272,7 +332,7 @@ inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const*for (int j = 0; j < 4; ++j) {__m512i g, sc;__mmask64 m;- Fmt<TY>::decode(blk, j, g, m, sc);+ FmtX<TY>::decode(blk, j, g, m, sc);for (int t = 0; t < NT; ++t) {const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j));const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);@@ -283,7 +343,7 @@ inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const*for (int t = 0; t < NT; ++t)accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold256(acci[t])), accf[t]);}- for (int t = 0; t < NT; ++t) res[t] = Fmt<TY>::K * hsum8_ggml(accf[t]);+ for (int t = 0; t < NT; ++t) res[t] = FmtX<TY>::K * hsum8_ggml(accf[t]);}template <int TY, int NT>@@ -325,7 +385,7 @@ void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const*for (int k = 0; k < rr * tt; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());for (int i = 0; i < nb; ++i)for (int q = 0; q < rr; ++q) {- const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * Fmt<TY>::bytes;+ const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * FmtX<TY>::bytes;if (prefetch_ahead > 0) {_mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);_mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);@@ -335,7 +395,7 @@ void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const*for (int j = 0; j < 4; ++j) {__m512i g, s;__mmask64 m;- Fmt<TY>::decode(blk, j, g, m, s);+ FmtX<TY>::decode(blk, j, g, m, s);w8[j] = _mm512_mask_sub_epi8(g, m, zero, g);sc[j] = s;corr = _mm512_dpwssd_epi32(corr, _mm512_maddubs_epi16(one8, w8[j]), s);@@ -355,7 +415,7 @@ void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const*}for (int q = 0; q < rr; ++q)for (int t = 0; t < tt; ++t)- out[t0 + t][rg + q] = Fmt<TY>::K * hsum8_ggml(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));+ out[t0 + t][rg + q] = FmtX<TY>::K * hsum8_ggml(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));}}}@@ -373,6 +433,110 @@ void dot_rows_x_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void}}++// ---- IQ1_M (29), AVX-512, 1..6 tokens: ggml's dot lane for lane. The grid indices come out of two pdep (as ggml's BMI2 path), the+// +-1 delta signs out of one pext into an 8-bit mask a 64-value step; the signed grid bytes are split into |g| and a negative mask;+// the delta dot is the pair sums of the activation (made once a call) times +-scale. d is f16 in the top nibbles of the scales.+inline void iq1m_pre(const block_q8_K* y, int nb, int16_t* p) {+ const __m256i one8 = _mm256_set1_epi8(1);+ for (int i = 0; i < nb; ++i)+ for (int c = 0; c < 8; ++c)+ _mm256_storeu_si256((__m256i*) (p + ((size_t) i * 8 + c) * 16),+ _mm256_maddubs_epi16(one8, _mm256_loadu_si256((const __m256i*) (y[i].qs + 32 * c))));+}++template <int NT>+void rows_m(const uint8_t* w, size_t rb, int n, const block_q8_K* const* y, const int16_t* const* P, float* const* out, int r0, int r1) {+ const int nb = n / QK_K;+ const __m256i mask = _mm256_set1_epi16(0x7), mone = _mm256_set1_epi16(1), mtwo8 = _mm256_set1_epi8(2);+ const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);+ const __m512i zero = _mm512_setzero_si512();+ for (int r = r0; r < r1; ++r) {+ __m256 a1[NT], a2[NT];+ for (int t = 0; t < NT; ++t) a1[t] = a2[t] = _mm256_setzero_ps();+ const uint8_t* row = w + (size_t) r * rb;+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = row + (size_t) i * 56;+ const uint8_t* qs = blk;+ const uint8_t* qh = blk + 32;+ const uint8_t* scb = blk + 48;+ if (prefetch_ahead > 0) {+ _mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);+ _mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);+ }+ const uint16_t sc0 = u16(scb), sc1 = u16(scb + 2), sc2 = u16(scb + 4), sc3 = u16(scb + 6);+ const uint16_t d16 = (uint16_t) ((sc0 >> 12) | ((sc1 >> 8) & 0x00f0) | ((sc2 >> 4) & 0x0f00) | (sc3 & 0xf000));+ __m256i scales = _mm256_set1_epi64x((long long) u64(scb));+ scales = _mm256_srlv_epi64(scales, scales_shift);+ scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);+ __m256i sidx1 = _mm256_set1_epi16(0x0100);+ __m256i sidx2 = _mm256_add_epi8(sidx1, _mm256_set1_epi8(8));+ __m512i s1[NT], s2[NT];+ for (int t = 0; t < NT; ++t) s1[t] = s2[t] = zero;+ for (int it = 0; it < 4; ++it) {+ const uint32_t qh32 = u32(qh + 4 * it);+ const uint64_t p1 = _pdep_u64(u32(qs + 8 * it), 0x00ff00ff00ff00ffULL) | _pdep_u64(qh32 & 0x7777, 0x0f000f000f000f00ULL);+ const uint64_t p2 = _pdep_u64(u32(qs + 8 * it + 4), 0x00ff00ff00ff00ffULL) | _pdep_u64((qh32 >> 16) & 0x7777, 0x0f000f000f000f00ULL);+ uint16_t ix[8];+ std::memcpy(ix, &p1, 8);+ std::memcpy(ix + 4, &p2, 8);+ const __m512i g = _mm512_set_epi64((long long) iq1s_grid[ix[7]], (long long) iq1s_grid[ix[6]], (long long) iq1s_grid[ix[5]],+ (long long) iq1s_grid[ix[4]], (long long) iq1s_grid[ix[3]], (long long) iq1s_grid[ix[2]],+ (long long) iq1s_grid[ix[1]], (long long) iq1s_grid[ix[0]]);+ const __m512i negv = _mm512_maskz_set1_epi64((__mmask8) _pext_u32(qh32, 0x88888888u), -1);+ const __m256i sca = _mm256_shuffle_epi8(scales, sidx1);+ const __m256i scb2 = _mm256_shuffle_epi8(scales, sidx2);+ sidx1 = _mm256_add_epi8(sidx1, mtwo8);+ sidx2 = _mm256_add_epi8(sidx2, mtwo8);+ const __m512i sc = _mm512_inserti64x4(_mm512_castsi256_si512(sca), scb2, 1);+ const __m512i ssc = _mm512_sub_epi16(_mm512_xor_si512(sc, negv), negv);+ const __m512i ag = _mm512_abs_epi8(g);+ const __mmask64 gneg = _mm512_movepi8_mask(g);+ for (int t = 0; t < NT; ++t) {+ const __m512i q8 = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * it));+ const __m512i pp = _mm512_loadu_si512((const void*) (P[t] + ((size_t) i * 8 + 2 * it) * 16));+ const __m512i ys = _mm512_mask_sub_epi8(q8, gneg, zero, q8);+ s1[t] = _mm512_dpwssd_epi32(s1[t], _mm512_maddubs_epi16(ag, ys), sc);+ s2[t] = _mm512_dpwssd_epi32(s2[t], pp, ssc);+ }+ }+ const float dx = h2f(d16);+ for (int t = 0; t < NT; ++t) {+ const __m256 d = _mm256_set1_ps(y[t][i].d * dx);+ a1[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(fold256(s1[t])), a1[t]);+ a2[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(fold256(s2[t])), a2[t]);+ }+ }+ for (int t = 0; t < NT; ++t) out[t][r] = hsum8_ggml(a1[t]) + 0.125f * hsum8_ggml(a2[t]);+ }+}++void rows_m_nt(int nt, const uint8_t* w, size_t rb, int n, const void* const* act, float* const* out, int r0, int r1) {+ const int nb = n / QK_K;+ static thread_local std::vector<int16_t> pre;+ for (int t0 = 0; t0 < nt; t0 += 6) {+ const int k = nt - t0 < 6 ? nt - t0 : 6;+ const block_q8_K* y[6];+ const int16_t* P[6];+ float* o[6];+ pre.resize((size_t) k * n);+ for (int t = 0; t < k; ++t) {+ y[t] = (const block_q8_K*) act[t0 + t];+ iq1m_pre(y[t], nb, pre.data() + (size_t) t * n);+ P[t] = pre.data() + (size_t) t * n;+ o[t] = out[t0 + t];+ }+ switch (k) {+ case 1: rows_m<1>(w, rb, n, y, P, o, r0, r1); break;+ case 2: rows_m<2>(w, rb, n, y, P, o, r0, r1); break;+ case 3: rows_m<3>(w, rb, n, y, P, o, r0, r1); break;+ case 4: rows_m<4>(w, rb, n, y, P, o, r0, r1); break;+ case 5: rows_m<5>(w, rb, n, y, P, o, r0, r1); break;+ default: rows_m<6>(w, rb, n, y, P, o, r0, r1); break;+ }+ }+}+} // namespacevoid iq512_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,@@ -399,7 +563,24 @@ void iq512_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*}}-bool iq512x_supported(int type) noexcept { return type == 16 || type == 17 || type == 18 || type == 21 || type == 22; }+static bool has_vpopcnt() {+ static const bool ok = [] {+#if defined(_MSC_VER)+ int r[4];+ __cpuidex(r, 7, 0);+ return ((r[2] >> 14) & 1) != 0; // AVX512_VPOPCNTDQ+#else+ unsigned a, b, c, d;+ __cpuid_count(7, 0, a, b, c, d);+ return ((c >> 14) & 1u) != 0;+#endif+ }();+ return ok;+}++bool iq512x_supported(int type) noexcept {+ return has_vpopcnt() && (type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 29);+}void iq512x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out,int r0, int r1) {@@ -409,6 +590,7 @@ void iq512x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const voidcase 18: dot_rows_x_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;case 21: dot_rows_x_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;case 22: dot_rows_x_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;+ case 29: rows_m_nt(nt, w, row_bytes, n, act, out, r0, r1); break;default: break;}}