Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 cpu lane: one-token AVX-512 rows - sign masks by popcount (IQ2_XXS, IQ2_XS, IQ3_XXS), IQ1_M AVX-512 rows with pdep indices; chosen per CPU (Zen 5 from 1 token)

Niko12211747c8e53882
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 80ba5890c..8a7894466 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1663,7 +1663,7 @@ if(STRATA_NATIVE_EXPERTS)
set_source_files_properties(src/kernels/cpu/iq_avx1.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX")
else()
set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp PROPERTIES COMPILE_OPTIONS
- "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512vnni;-mavx512vbmi;-mf16c;-mfma")
+ "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512vnni;-mavx512vbmi;-mavx512vpopcntdq;-mbmi2;-mf16c;-mfma")
set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cpp
index b8da9626f..a54a22902 100644
--- a/src/dsv41/ds4_gg_cpu.cpp
+++ b/src/dsv41/ds4_gg_cpu.cpp
@@ -183,15 +183,16 @@ static bool want_512(int type, int nt) {
}
if (g_iq512 == 0 || !strata::kernels::cpu::iq512x_supported(type)) return false;
if (g_iq512 == 1) return true;
- // by CPU (measured, ds4_gg_bench --expert at 16 tokens; the AVX-512 rows tie the AVX2 ones at 4 tokens and lose at one):
- // Zen 5 (AMD family 0x1A, Strix Halo): IQ2_XXS 8.1 -> 10.9 GB/s, IQ3_XXS 10.8 -> 13.9;
- // Zen 4 (family 0x19, Ryzen 5 7600): IQ2_XXS 3.2 -> 4.0, IQ3_XXS no gain;
- // anything else with AVX-512 (Intel): not measured, so the AVX2 rows.
- // and only for a prompt chunk's tokens (7 and more)
- if (nt < 7) return false;
+ // by CPU (measured, ds4_gg_bench --expert; the rows are bit-identical to ggml's dot everywhere):
+ // Zen 5 (AMD family 0x1A, Strix Halo, 16 threads, GB/s of weights ggml -> AVX-512 rows): one token IQ2_XXS 35 -> 44, IQ2_XS 37 -> 48,
+ // IQ1_M 32 -> 36 (the sign masks by popcount, IQ1_M's indices by pdep: no table loads), IQ3_XXS 43 -> 45; 4 tokens x2.5-3.5; 16 tokens x4.4-4.8;
+ // Zen 4 (family 0x19): see the Zen 4 table; anything else with AVX-512 (Intel): not measured, so the AVX2 rows
const int fam = amd_family();
- if (fam >= 0x1A) return type == 16 || type == 18;
- if (fam == 0x19) return type == 16;
+ if (fam >= 0x1A) {
+ if (type == 16 || type == 17 || type == 29) return true;
+ return type == 18 && nt >= 2;
+ }
+ if (fam == 0x19) return nt >= 7 && type == 16;
return false;
}
// 0: ggml-cpu's dot, 1: the AVX2 rows, 2: the AVX-512 rows
diff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cpp
index b7c6e56ff..14330f6ec 100644
--- a/src/kernels/cpu/iq_avx512.cpp
+++ b/src/kernels/cpu/iq_avx512.cpp
@@ -15,6 +15,11 @@
#include "ggml-common.h"
#include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
#include <cmath>
#include <cstdlib>
@@ -242,6 +247,61 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*
}
+// The sign masks of IQ2_XXS / IQ2_XS / IQ3_XXS come from a 7-bit index whose 8th bit is the parity of the other seven (ksigns_iq2xs[i] =
+// i | parity(i) << 7). The table needs a load and a few shifts and ors a group of eight values; popcount does it for all eight groups
+// of 64 values in one vector: measured on Zen 5, one-token IQ2_XXS rows 33 -> 45-50 GB/s (16 threads, DRAM-sized), cache-resident 16.8 -> 13.4 ns a block.
+// (AVX512_VPOPCNTDQ + VL: Zen 4, Zen 5, Ice Lake and later; iq512x_supported() asks for it.)
+inline __mmask64 sign_mask_from_idx(__m256i si) {
+ const __m256i par = _mm256_slli_epi32(_mm256_and_si256(_mm256_popcnt_epi32(si), _mm256_set1_epi32(1)), 7);
+ return _cvtu64_mask64((uint64_t) _mm_cvtsi128_si64(_mm256_cvtepi32_epi8(_mm256_or_si256(si, par))));
+}
+inline __m256i sign_idx_xxs(uint32_t lo4, uint32_t hi4) { // four 7-bit fields of each of two words -> eight dwords
+ const __m256i w = _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_set1_epi32((int) lo4)), _mm_set1_epi32((int) hi4), 1);
+ return _mm256_and_si256(_mm256_srlv_epi32(w, _mm256_setr_epi32(0, 7, 14, 21, 0, 7, 14, 21)), _mm256_set1_epi32(127));
+}
+
+template <int TY> struct FmtX : Fmt<TY> {};
+template <> struct FmtX<16> : Fmt<16> {
+ static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
+ const uint8_t* q = b + 2 + 16 * j;
+ const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);
+ g = _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],
+ (long long) iq2xxs_grid[(b0 >> 8) & 255], (long long) iq2xxs_grid[b0 & 255],
+ (long long) iq2xxs_grid[a0 >> 24], (long long) iq2xxs_grid[(a0 >> 16) & 255],
+ (long long) iq2xxs_grid[(a0 >> 8) & 255], (long long) iq2xxs_grid[a0 & 255]);
+ m = sign_mask_from_idx(sign_idx_xxs(a1, b1));
+ const int sa = 2 * (int) (a1 >> 28) + 1, sb = 2 * (int) (b1 >> 28) + 1;
+ sc = scales4(sa, sa, sb, sb);
+ }
+};
+template <> struct FmtX<17> : Fmt<17> {
+ static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
+ const uint8_t* q = b + 2 + 16 * j;
+ uint16_t v[8];
+ std::memcpy(v, q, 16);
+ g = _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],
+ (long long) iq2xs_grid[v[5] & 511], (long long) iq2xs_grid[v[4] & 511],
+ (long long) iq2xs_grid[v[3] & 511], (long long) iq2xs_grid[v[2] & 511],
+ (long long) iq2xs_grid[v[1] & 511], (long long) iq2xs_grid[v[0] & 511]);
+ m = sign_mask_from_idx(_mm256_srli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*) q)), 9));
+ const uint8_t s0 = b[66 + 2 * j], s1 = b[66 + 2 * j + 1];
+ sc = scales4(2 * (s0 & 15) + 1, 2 * (s0 >> 4) + 1, 2 * (s1 & 15) + 1, 2 * (s1 >> 4) + 1);
+ }
+};
+template <> struct FmtX<18> : Fmt<18> {
+ static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
+ const uint8_t* q = b + 2 + 16 * j;
+ g = _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
+ (int) iq3xxs_grid[q[11]], (int) iq3xxs_grid[q[10]], (int) iq3xxs_grid[q[9]], (int) iq3xxs_grid[q[8]],
+ (int) iq3xxs_grid[q[7]], (int) iq3xxs_grid[q[6]], (int) iq3xxs_grid[q[5]], (int) iq3xxs_grid[q[4]],
+ (int) iq3xxs_grid[q[3]], (int) iq3xxs_grid[q[2]], (int) iq3xxs_grid[q[1]], (int) iq3xxs_grid[q[0]]);
+ const uint32_t a = u32(b + 2 + 64 + 8 * j), c = u32(b + 2 + 64 + 8 * j + 4);
+ m = sign_mask_from_idx(sign_idx_xxs(a, c));
+ const int sa = 2 * (int) (a >> 28) + 1, sb = 2 * (int) (c >> 28) + 1;
+ sc = scales4(sa, sa, sb, sb);
+ }
+};
+
// ---- the DeepSeek CPU lane's exact rows (iq512x_rows): the same 64-value decode, but summed as ggml's AVX2 dots sum, so a
// token's row is ggml-cpu's vec_dot bit for bit at any number of tokens: the integer sums of a block fold to ggml's eight lanes
// (low half = the even 32-value chunk, high half = the odd one), the block scale is f16(d) * y.d, eight float lanes accumulate
@@ -261,7 +321,7 @@ inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const*
for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
const __m512i zero = _mm512_setzero_si512();
for (int i = 0; i < nblocks; ++i) {
- const uint8_t* blk = row + (size_t) i * Fmt<TY>::bytes;
+ const uint8_t* blk = row + (size_t) i * FmtX<TY>::bytes;
if (prefetch_ahead > 0) {
const uint8_t* ahead = blk + (size_t) prefetch_ahead;
_mm_prefetch((const char*) ahead, _MM_HINT_T0);
@@ -272,7 +332,7 @@ inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const*
for (int j = 0; j < 4; ++j) {
__m512i g, sc;
__mmask64 m;
- Fmt<TY>::decode(blk, j, g, m, sc);
+ FmtX<TY>::decode(blk, j, g, m, sc);
for (int t = 0; t < NT; ++t) {
const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j));
const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);
@@ -283,7 +343,7 @@ inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const*
for (int t = 0; t < NT; ++t)
accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold256(acci[t])), accf[t]);
}
- for (int t = 0; t < NT; ++t) res[t] = Fmt<TY>::K * hsum8_ggml(accf[t]);
+ for (int t = 0; t < NT; ++t) res[t] = FmtX<TY>::K * hsum8_ggml(accf[t]);
}
template <int TY, int NT>
@@ -325,7 +385,7 @@ void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const*
for (int k = 0; k < rr * tt; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());
for (int i = 0; i < nb; ++i)
for (int q = 0; q < rr; ++q) {
- const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * Fmt<TY>::bytes;
+ const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * FmtX<TY>::bytes;
if (prefetch_ahead > 0) {
_mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);
_mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);
@@ -335,7 +395,7 @@ void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const*
for (int j = 0; j < 4; ++j) {
__m512i g, s;
__mmask64 m;
- Fmt<TY>::decode(blk, j, g, m, s);
+ FmtX<TY>::decode(blk, j, g, m, s);
w8[j] = _mm512_mask_sub_epi8(g, m, zero, g);
sc[j] = s;
corr = _mm512_dpwssd_epi32(corr, _mm512_maddubs_epi16(one8, w8[j]), s);
@@ -355,7 +415,7 @@ void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const*
}
for (int q = 0; q < rr; ++q)
for (int t = 0; t < tt; ++t)
- out[t0 + t][rg + q] = Fmt<TY>::K * hsum8_ggml(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));
+ out[t0 + t][rg + q] = FmtX<TY>::K * hsum8_ggml(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));
}
}
}
@@ -373,6 +433,110 @@ void dot_rows_x_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void
}
}
+
+// ---- IQ1_M (29), AVX-512, 1..6 tokens: ggml's dot lane for lane. The grid indices come out of two pdep (as ggml's BMI2 path), the
+// +-1 delta signs out of one pext into an 8-bit mask a 64-value step; the signed grid bytes are split into |g| and a negative mask;
+// the delta dot is the pair sums of the activation (made once a call) times +-scale. d is f16 in the top nibbles of the scales.
+inline void iq1m_pre(const block_q8_K* y, int nb, int16_t* p) {
+ const __m256i one8 = _mm256_set1_epi8(1);
+ for (int i = 0; i < nb; ++i)
+ for (int c = 0; c < 8; ++c)
+ _mm256_storeu_si256((__m256i*) (p + ((size_t) i * 8 + c) * 16),
+ _mm256_maddubs_epi16(one8, _mm256_loadu_si256((const __m256i*) (y[i].qs + 32 * c))));
+}
+
+template <int NT>
+void rows_m(const uint8_t* w, size_t rb, int n, const block_q8_K* const* y, const int16_t* const* P, float* const* out, int r0, int r1) {
+ const int nb = n / QK_K;
+ const __m256i mask = _mm256_set1_epi16(0x7), mone = _mm256_set1_epi16(1), mtwo8 = _mm256_set1_epi8(2);
+ const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);
+ const __m512i zero = _mm512_setzero_si512();
+ for (int r = r0; r < r1; ++r) {
+ __m256 a1[NT], a2[NT];
+ for (int t = 0; t < NT; ++t) a1[t] = a2[t] = _mm256_setzero_ps();
+ const uint8_t* row = w + (size_t) r * rb;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t* blk = row + (size_t) i * 56;
+ const uint8_t* qs = blk;
+ const uint8_t* qh = blk + 32;
+ const uint8_t* scb = blk + 48;
+ if (prefetch_ahead > 0) {
+ _mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);
+ _mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);
+ }
+ const uint16_t sc0 = u16(scb), sc1 = u16(scb + 2), sc2 = u16(scb + 4), sc3 = u16(scb + 6);
+ const uint16_t d16 = (uint16_t) ((sc0 >> 12) | ((sc1 >> 8) & 0x00f0) | ((sc2 >> 4) & 0x0f00) | (sc3 & 0xf000));
+ __m256i scales = _mm256_set1_epi64x((long long) u64(scb));
+ scales = _mm256_srlv_epi64(scales, scales_shift);
+ scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);
+ __m256i sidx1 = _mm256_set1_epi16(0x0100);
+ __m256i sidx2 = _mm256_add_epi8(sidx1, _mm256_set1_epi8(8));
+ __m512i s1[NT], s2[NT];
+ for (int t = 0; t < NT; ++t) s1[t] = s2[t] = zero;
+ for (int it = 0; it < 4; ++it) {
+ const uint32_t qh32 = u32(qh + 4 * it);
+ const uint64_t p1 = _pdep_u64(u32(qs + 8 * it), 0x00ff00ff00ff00ffULL) | _pdep_u64(qh32 & 0x7777, 0x0f000f000f000f00ULL);
+ const uint64_t p2 = _pdep_u64(u32(qs + 8 * it + 4), 0x00ff00ff00ff00ffULL) | _pdep_u64((qh32 >> 16) & 0x7777, 0x0f000f000f000f00ULL);
+ uint16_t ix[8];
+ std::memcpy(ix, &p1, 8);
+ std::memcpy(ix + 4, &p2, 8);
+ const __m512i g = _mm512_set_epi64((long long) iq1s_grid[ix[7]], (long long) iq1s_grid[ix[6]], (long long) iq1s_grid[ix[5]],
+ (long long) iq1s_grid[ix[4]], (long long) iq1s_grid[ix[3]], (long long) iq1s_grid[ix[2]],
+ (long long) iq1s_grid[ix[1]], (long long) iq1s_grid[ix[0]]);
+ const __m512i negv = _mm512_maskz_set1_epi64((__mmask8) _pext_u32(qh32, 0x88888888u), -1);
+ const __m256i sca = _mm256_shuffle_epi8(scales, sidx1);
+ const __m256i scb2 = _mm256_shuffle_epi8(scales, sidx2);
+ sidx1 = _mm256_add_epi8(sidx1, mtwo8);
+ sidx2 = _mm256_add_epi8(sidx2, mtwo8);
+ const __m512i sc = _mm512_inserti64x4(_mm512_castsi256_si512(sca), scb2, 1);
+ const __m512i ssc = _mm512_sub_epi16(_mm512_xor_si512(sc, negv), negv);
+ const __m512i ag = _mm512_abs_epi8(g);
+ const __mmask64 gneg = _mm512_movepi8_mask(g);
+ for (int t = 0; t < NT; ++t) {
+ const __m512i q8 = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * it));
+ const __m512i pp = _mm512_loadu_si512((const void*) (P[t] + ((size_t) i * 8 + 2 * it) * 16));
+ const __m512i ys = _mm512_mask_sub_epi8(q8, gneg, zero, q8);
+ s1[t] = _mm512_dpwssd_epi32(s1[t], _mm512_maddubs_epi16(ag, ys), sc);
+ s2[t] = _mm512_dpwssd_epi32(s2[t], pp, ssc);
+ }
+ }
+ const float dx = h2f(d16);
+ for (int t = 0; t < NT; ++t) {
+ const __m256 d = _mm256_set1_ps(y[t][i].d * dx);
+ a1[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(fold256(s1[t])), a1[t]);
+ a2[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(fold256(s2[t])), a2[t]);
+ }
+ }
+ for (int t = 0; t < NT; ++t) out[t][r] = hsum8_ggml(a1[t]) + 0.125f * hsum8_ggml(a2[t]);
+ }
+}
+
+void rows_m_nt(int nt, const uint8_t* w, size_t rb, int n, const void* const* act, float* const* out, int r0, int r1) {
+ const int nb = n / QK_K;
+ static thread_local std::vector<int16_t> pre;
+ for (int t0 = 0; t0 < nt; t0 += 6) {
+ const int k = nt - t0 < 6 ? nt - t0 : 6;
+ const block_q8_K* y[6];
+ const int16_t* P[6];
+ float* o[6];
+ pre.resize((size_t) k * n);
+ for (int t = 0; t < k; ++t) {
+ y[t] = (const block_q8_K*) act[t0 + t];
+ iq1m_pre(y[t], nb, pre.data() + (size_t) t * n);
+ P[t] = pre.data() + (size_t) t * n;
+ o[t] = out[t0 + t];
+ }
+ switch (k) {
+ case 1: rows_m<1>(w, rb, n, y, P, o, r0, r1); break;
+ case 2: rows_m<2>(w, rb, n, y, P, o, r0, r1); break;
+ case 3: rows_m<3>(w, rb, n, y, P, o, r0, r1); break;
+ case 4: rows_m<4>(w, rb, n, y, P, o, r0, r1); break;
+ case 5: rows_m<5>(w, rb, n, y, P, o, r0, r1); break;
+ default: rows_m<6>(w, rb, n, y, P, o, r0, r1); break;
+ }
+ }
+}
+
} // namespace
void iq512_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,
@@ -399,7 +563,24 @@ void iq512_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*
}
}
-bool iq512x_supported(int type) noexcept { return type == 16 || type == 17 || type == 18 || type == 21 || type == 22; }
+static bool has_vpopcnt() {
+ static const bool ok = [] {
+#if defined(_MSC_VER)
+ int r[4];
+ __cpuidex(r, 7, 0);
+ return ((r[2] >> 14) & 1) != 0; // AVX512_VPOPCNTDQ
+#else
+ unsigned a, b, c, d;
+ __cpuid_count(7, 0, a, b, c, d);
+ return ((c >> 14) & 1u) != 0;
+#endif
+ }();
+ return ok;
+}
+
+bool iq512x_supported(int type) noexcept {
+ return has_vpopcnt() && (type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 29);
+}
void iq512x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out,
int r0, int r1) {
@@ -409,6 +590,7 @@ void iq512x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void
case 18: dot_rows_x_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;
case 21: dot_rows_x_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;
case 22: dot_rows_x_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;
+ case 29: rows_m_nt(nt, w, row_bytes, n, act, out, r0, r1); break;
default: break;
}
}