Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

CPU expert rows: 512-bit grid lookups by gather where a gather is cheap; one IQ3 token on the exact 512-bit rows

Niko122116c4da9443fb
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7dd95dfdf..41d9fd863 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1776,6 +1776,14 @@ if(STRATA_NATIVE_EXPERTS)
# against ggml-cpu's dot, on random blocks; --bench: ms per expert on one thread, the weights streamed from DRAM
add_executable(iq_avx2_parity src/kernels/cpu/iq_avx2_parity.cpp)
target_link_libraries(iq_avx2_parity PRIVATE strata_kernels_cpu)
+ # the 512-bit i-quant rows against ggml-cpu's dot: the exact rows bit for bit at 1-4 tokens, the engine's multi-token rows to float
+ # rounding, with the grid lookups as scalar loads and as gathers (SKIP without AVX-512)
+ add_executable(iq512_gather_parity src/kernels/cpu/iq512_gather_parity.cpp)
+ target_link_libraries(iq512_gather_parity PRIVATE strata_kernels_cpu)
+ if(STRATA_BUILD_TESTS)
+ add_test(NAME iq512_gather_parity_scalar COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ_GATHER=0 $<TARGET_FILE:iq512_gather_parity>)
+ add_test(NAME iq512_gather_parity_gather COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ_GATHER=1 $<TARGET_FILE:iq512_gather_parity>)
+ endif()
# MODEL-UPDATE: the pool's expert rows per format on T threads against the RAM bandwidth (a benchmark, no test)
add_executable(kq_cpu_bench src/kernels/cpu/kq_cpu_bench.cpp)
target_link_libraries(kq_cpu_bench PRIVATE strata_kernels_cpu)
diff --git a/include/strata/kernels/cpu/iq_avx512.hpp b/include/strata/kernels/cpu/iq_avx512.hpp
index ae21dfd44..3f51b9dee 100644
--- a/include/strata/kernels/cpu/iq_avx512.hpp
+++ b/include/strata/kernels/cpu/iq_avx512.hpp
@@ -8,6 +8,9 @@
namespace strata::kernels::cpu {
bool iq512_supported(int ggml_type) noexcept;
+/// Whether the grid lookups of these rows are gathers (the default where cpu_gather_fast(), STRATA_IQ_GATHER=1 / 0 force it): the
+/// same words, so the same bits as the scalar lookups, and 1.3-2x faster where a gather is cheap (Sapphire Rapids).
+bool iq512_gather_on() noexcept;
/// ff[t][r] = silu(gate_r . a[t]) * (up_r . a[t]), rows [r0, r1); gate rows at blob, up rows at blob + up_off.
void iq512_gu_rows(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act,
int nt, float* const* ff, int r0, int r1);
diff --git a/src/kernels/cpu/iq512_gather_parity.cpp b/src/kernels/cpu/iq512_gather_parity.cpp
new file mode 100644
index 000000000..502880ea0
--- /dev/null
+++ b/src/kernels/cpu/iq512_gather_parity.cpp
@@ -0,0 +1,76 @@
+// iq512_gather_parity: the 512-bit i-quant rows (iq_avx512.cpp) against ggml-cpu's own dot.
+// (a) iq512x_rows, for 1..4 tokens, is ggml-cpu's single-token vec_dot bit for bit (IQ3_XXS, IQ3_S, IQ2_S), whatever the
+// grid lookup is (scalar loads or gathers: STRATA_IQ_GATHER=0 / 1, ctest runs both);
+// (b) iq512_rows (the engine's multi-token rows) agrees with it to float rounding (another summation order).
+// Synthetic blocks (random bytes, a fixed f16 scale), so no quantizer and no imatrix are needed. SKIP without AVX-512.
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/iq_avx512.hpp"
+
+#include "ggml.h"
+#include "ggml-cpu.h"
+
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+namespace cpu = strata::kernels::cpu;
+
+static int g_fail = 0;
+#define CHECK(c) \
+ do { \
+ if (!(c)) { std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #c); ++g_fail; } \
+ } while (0)
+
+int main() {
+ ggml_cpu_init();
+ if (!cpu::cpu_avx512_ok()) {
+ std::printf("iq512_gather_parity: SKIP (no AVX-512)\n");
+ return 0;
+ }
+ std::printf("iq512_gather_parity: gather %d\n", (int) cpu::iq512_gather_on());
+ constexpr int n = 2560, rows = 96, ntmax = 4;
+ const ggml_type_traits_cpu* t8 = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K);
+ uint32_t seed = 987654321u;
+ auto rnd = [&seed] { seed = seed * 1664525u + 1013904223u; return seed >> 8; };
+ const size_t ab = ggml_row_size(GGML_TYPE_Q8_K, n);
+ std::vector<std::vector<uint8_t>> act(ntmax, std::vector<uint8_t>(ab));
+ const void* ap[ntmax];
+ for (int t = 0; t < ntmax; ++t) {
+ std::vector<float> x(n);
+ for (auto& v : x) v = ((float) (rnd() % 2001) - 1000.f) * 0.001f * (1.f + (float) t);
+ t8->from_float(x.data(), act[t].data(), n);
+ ap[t] = act[t].data();
+ }
+ for (int type : {18, 21, 22}) {
+ const ggml_type_traits_cpu* tr = ggml_get_type_traits_cpu((ggml_type) type);
+ CHECK(tr != nullptr && tr->vec_dot != nullptr);
+ CHECK(cpu::iq512x_supported(type) && cpu::iq512_supported(type));
+ const size_t rb = ggml_row_size((ggml_type) type, n), nb = (size_t) n / 256, bb = rb / nb;
+ std::vector<uint8_t> w((size_t) rows * rb);
+ for (auto& b : w) b = (uint8_t) rnd();
+ for (int r = 0; r < rows; ++r)
+ for (size_t k = 0; k < nb; ++k) { uint8_t* d = &w[(size_t) r * rb + k * bb]; d[0] = 0x66; d[1] = 0x2e; } // f16 ~0.1
+ std::vector<float> want((size_t) ntmax * rows), gx((size_t) ntmax * rows), ge((size_t) ntmax * rows);
+ for (int t = 0; t < ntmax; ++t)
+ for (int r = 0; r < rows; ++r)
+ tr->vec_dot(n, &want[(size_t) t * rows + r], 0, w.data() + (size_t) r * rb, 0, act[t].data(), 0, 1);
+ for (int nt = 1; nt <= ntmax; ++nt) {
+ float* ox[ntmax];
+ float* oe[ntmax];
+ for (int t = 0; t < ntmax; ++t) { ox[t] = &gx[(size_t) t * rows]; oe[t] = &ge[(size_t) t * rows]; }
+ cpu::iq512x_rows(type, w.data(), rb, n, ap, nt, ox, 0, rows);
+ cpu::iq512_rows(type, w.data(), rb, n, ap, nt, oe, 0, rows);
+ for (int t = 0; t < nt; ++t)
+ for (int r = 0; r < rows; ++r) {
+ const float want_v = want[(size_t) t * rows + r];
+ CHECK(std::memcmp(&gx[(size_t) t * rows + r], &want_v, 4) == 0); // (a) bit for bit
+ const float e = ge[(size_t) t * rows + r];
+ CHECK(std::fabs(e - want_v) <= 1e-3f * std::fabs(want_v) + 1e-3f); // (b) float-rounding agreement
+ }
+ }
+ std::printf(" type %d: ok\n", type);
+ }
+ if (g_fail == 0) std::printf("iq512_gather_parity: OK\n");
+ return g_fail == 0 ? 0 : 1;
+}
diff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cpp
index 14330f6ec..aaa0adaa3 100644
--- a/src/kernels/cpu/iq_avx512.cpp
+++ b/src/kernels/cpu/iq_avx512.cpp
@@ -9,6 +9,7 @@
// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22). IQ1_M stays on ggml-cpu (no shipped
// model has IQ1_M expert rows: the 'Coder IQ1_M' pack's gate/up are IQ2_S / IQ3_XXS / IQ3_S).
#include "strata/kernels/cpu/iq_avx512.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
#define GGML_COMMON_DECL_CPP
#define GGML_COMMON_IMPL_CPP
@@ -36,7 +37,14 @@ inline uint64_t u64(const uint8_t* p) { uint64_t v; std::memcpy(&v, p, 8); retur
// E-2: the IQ3 grids by one AVX-512 gather (the same words, the same results) - measured 3-5% SLOWER than the scalar
// lookups on a Ryzen 5 7600 (Zen 4 gathers are microcoded), so opt-in: STRATA_IQ_GATHER=1
-const bool gather = std::getenv("STRATA_IQ_GATHER") != nullptr;
+// 0.2.5: DEFAULT on where gathers are fast (Intel server and HEDT cores with AVX-512: cpu_gather_fast), off on Zen 4 (microcoded);
+// STRATA_IQ_GATHER=1 / 0 force it either way. The same words are loaded, so the results are bit for bit the scalar decode's.
+const bool gather = [] {
+ const char* v = std::getenv("STRATA_IQ_GATHER");
+ return v != nullptr && v[0] != '\0' ? std::atoi(v) != 0 : cpu_gather_fast();
+}();
+// the grid lookups by one 64-bit gather of eight entries (IQ2_XXS / IQ2_XS / IQ2_S): `idx` = eight 32-bit indices
+inline __m512i gather64(__m256i idx, const void* grid) { return _mm512_i32gather_epi64(idx, grid, 8); }
// E-2: software prefetch distance in bytes (STRATA_IQ_PREFETCH; 0 = off): -2-3% gate/up time on IQ3_S decode
// (the rows are read at ~30 GB/s by 6 cores, near what DDR5 with 4 KB pages gives: the pool is memory-bound)
const int prefetch_ahead = [] {
@@ -62,7 +70,8 @@ template <> struct Fmt<16> { // IQ2_XXS: d, qs[32] u16
static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
const uint8_t* q = b + 2 + 16 * j;
const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);
- g = _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],
+ g = gather ? gather64(_mm256_cvtepu8_epi32(_mm_cvtsi64_si128((long long) ((uint64_t) a0 | ((uint64_t) b0 << 32)))), iq2xxs_grid)
+ : _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],
(long long) iq2xxs_grid[(b0 >> 8) & 255], (long long) iq2xxs_grid[b0 & 255],
(long long) iq2xxs_grid[a0 >> 24], (long long) iq2xxs_grid[(a0 >> 16) & 255],
(long long) iq2xxs_grid[(a0 >> 8) & 255], (long long) iq2xxs_grid[a0 & 255]);
@@ -83,7 +92,8 @@ template <> struct Fmt<17> { // IQ2_XS: d, qs[32] u16 (9-bit grid index + 7-bi
const uint8_t* q = b + 2 + 16 * j;
uint16_t v[8];
std::memcpy(v, q, 16);
- g = _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],
+ g = gather ? gather64(_mm256_and_si256(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*) q)), _mm256_set1_epi32(511)), iq2xs_grid)
+ : _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],
(long long) iq2xs_grid[v[5] & 511], (long long) iq2xs_grid[v[4] & 511],
(long long) iq2xs_grid[v[3] & 511], (long long) iq2xs_grid[v[2] & 511],
(long long) iq2xs_grid[v[1] & 511], (long long) iq2xs_grid[v[0] & 511]);
@@ -101,6 +111,11 @@ template <> struct Fmt<22> { // IQ2_S: d, qs[64] (32 grid bytes, 32 sign bytes
static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
const uint8_t* qs = b + 2 + 8 * j;
const uint8_t h0 = b[66 + 2 * j], h1 = b[66 + 2 * j + 1];
+ if (gather) { // the 10-bit indices: the byte plus two high bits from qh (h0 for lanes 0-3, h1 for 4-7), one 64-bit gather
+ const __m256i hi = _mm256_and_si256(_mm256_sllv_epi32(_mm256_setr_epi32(h0, h0, h0, h0, h1, h1, h1, h1),
+ _mm256_setr_epi32(8, 6, 4, 2, 8, 6, 4, 2)), _mm256_set1_epi32(0x300));
+ g = gather64(_mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) qs)), hi), iq2s_grid);
+ } else
g = _mm512_set_epi64((long long) iq2s_grid[qs[7] | ((h1 << 2) & 0x300)], (long long) iq2s_grid[qs[6] | ((h1 << 4) & 0x300)],
(long long) iq2s_grid[qs[5] | ((h1 << 6) & 0x300)], (long long) iq2s_grid[qs[4] | ((h1 << 8) & 0x300)],
(long long) iq2s_grid[qs[3] | ((h0 << 2) & 0x300)], (long long) iq2s_grid[qs[2] | ((h0 << 4) & 0x300)],
@@ -265,7 +280,8 @@ template <> struct FmtX<16> : Fmt<16> {
static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
const uint8_t* q = b + 2 + 16 * j;
const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);
- g = _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],
+ g = gather ? gather64(_mm256_cvtepu8_epi32(_mm_cvtsi64_si128((long long) ((uint64_t) a0 | ((uint64_t) b0 << 32)))), iq2xxs_grid)
+ : _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],
(long long) iq2xxs_grid[(b0 >> 8) & 255], (long long) iq2xxs_grid[b0 & 255],
(long long) iq2xxs_grid[a0 >> 24], (long long) iq2xxs_grid[(a0 >> 16) & 255],
(long long) iq2xxs_grid[(a0 >> 8) & 255], (long long) iq2xxs_grid[a0 & 255]);
@@ -279,7 +295,8 @@ template <> struct FmtX<17> : Fmt<17> {
const uint8_t* q = b + 2 + 16 * j;
uint16_t v[8];
std::memcpy(v, q, 16);
- g = _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],
+ g = gather ? gather64(_mm256_and_si256(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*) q)), _mm256_set1_epi32(511)), iq2xs_grid)
+ : _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],
(long long) iq2xs_grid[v[5] & 511], (long long) iq2xs_grid[v[4] & 511],
(long long) iq2xs_grid[v[3] & 511], (long long) iq2xs_grid[v[2] & 511],
(long long) iq2xs_grid[v[1] & 511], (long long) iq2xs_grid[v[0] & 511]);
@@ -291,7 +308,8 @@ template <> struct FmtX<17> : Fmt<17> {
template <> struct FmtX<18> : Fmt<18> {
static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
const uint8_t* q = b + 2 + 16 * j;
- g = _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
+ g = gather ? _mm512_i32gather_epi32(_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*) q)), iq3xxs_grid, 4)
+ : _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
(int) iq3xxs_grid[q[11]], (int) iq3xxs_grid[q[10]], (int) iq3xxs_grid[q[9]], (int) iq3xxs_grid[q[8]],
(int) iq3xxs_grid[q[7]], (int) iq3xxs_grid[q[6]], (int) iq3xxs_grid[q[5]], (int) iq3xxs_grid[q[4]],
(int) iq3xxs_grid[q[3]], (int) iq3xxs_grid[q[2]], (int) iq3xxs_grid[q[1]], (int) iq3xxs_grid[q[0]]);
@@ -578,6 +596,8 @@ static bool has_vpopcnt() {
return ok;
}
+bool iq512_gather_on() noexcept { return gather; }
+
bool iq512x_supported(int type) noexcept {
return has_vpopcnt() && (type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 29);
}
diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cpp
index 7c491832c..ec1a68704 100644
--- a/src/kernels/cpu/native_expert.cpp
+++ b/src/kernels/cpu/native_expert.cpp
@@ -39,6 +39,40 @@ bool tq_rows_own(int type) {
return type >= (int) GGML_TYPE_COUNT || traits(type) == nullptr || traits(type)->vec_dot == nullptr;
}
+// 0.2.5: the 512-bit exact rows (iq512x_rows) take one token of an IQ3_XXS / IQ3_S expert only if this build's ggml-cpu dot gives
+// the same bits on synthetic blocks of that type (a ggml built for AVX-512 may carry another summation order): checked once a type.
+bool x512_one_ok(int type) {
+ static std::mutex mu;
+ static std::map<int, bool> memo;
+ std::lock_guard<std::mutex> lk(mu);
+ if (const auto it = memo.find(type); it != memo.end()) return it->second;
+ bool ok = false;
+ const ggml_type_traits_cpu* tr = traits(type);
+ const ggml_type_traits_cpu* t8 = traits(GGML_TYPE_Q8_K);
+ if (tr != nullptr && tr->vec_dot != nullptr && t8 != nullptr && t8->from_float != nullptr && iq512x_supported(type)) {
+ constexpr int n = 2560, rows = 24;
+ const size_t rb = ggml_row_size((ggml_type) type, n), nb = (size_t) n / 256, bb = rb / nb;
+ std::vector<uint8_t> w((size_t) rows * rb), act(ggml_row_size(GGML_TYPE_Q8_K, n));
+ uint32_t seed = 12345u;
+ auto rnd = [&seed] { seed = seed * 1664525u + 1013904223u; return seed >> 8; };
+ for (auto& b : w) b = (uint8_t) rnd();
+ for (int r = 0; r < rows; ++r)
+ for (size_t k = 0; k < nb; ++k) { uint8_t* d = &w[(size_t) r * rb + k * bb]; d[0] = 0x66; d[1] = 0x2e; } // d = ~0.1 (f16)
+ std::vector<float> x(n);
+ for (auto& v : x) v = ((float) (rnd() % 2001) - 1000.f) * 0.001f;
+ t8->from_float(x.data(), act.data(), n);
+ std::vector<float> got((size_t) rows), want((size_t) rows);
+ const void* ap[1] = {act.data()};
+ float* op[1] = {got.data()};
+ iq512x_rows(type, w.data(), rb, n, ap, 1, op, 0, rows);
+ for (int r = 0; r < rows; ++r) tr->vec_dot(n, &want[(size_t) r], 0, w.data() + (size_t) r * rb, 0, act.data(), 0, 1);
+ ok = std::memcmp(got.data(), want.data(), got.size() * sizeof(float)) == 0;
+ if (!ok) std::fprintf(stderr, "strata: the 512-bit exact one-token rows of type %d differ from this ggml-cpu's dot: not used\n", type);
+ }
+ memo[type] = ok;
+ return ok;
+}
+
void init_once() {
static std::once_flag once;
std::call_once(once, [] { ggml_cpu_init(); });
@@ -299,6 +333,25 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*
iq256_gu_rows_exact_one(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act[0], ff[0], r0, r1);
return;
}
+ // 0.2.5 (AVX-512 CPUs where a gather is cheap: Intel Sapphire Rapids and later): ONE token of an IQ3_XXS / IQ3_S expert on the
+ // 512-bit exact rows, ggml-cpu's single-token AVX2 dot bit for bit with the grid words gathered: 1.3x ggml's dot on a Xeon 8468
+ // (x512_bench), the same output. Where the multi-token rule (#152) already sends one token to the multi-token kernel
+ // (STRATA_IQ_MT_MIN=1) that stays; STRATA_NO_IQ512X1=1 turns this off; the first use checks the bits against this ggml.
+ static const bool x512_one = avx512 && iq512_gather_on() && std::getenv("STRATA_NO_IQ512X1") == nullptr;
+ if (x512_one && nt == 1 && nt < native_gu_mt_min(f.gu_type) && f.tail_off == 0 && (f.gu_type == 18 || f.gu_type == 21) &&
+ x512_one_ok(f.gu_type)) {
+ const int n1 = (int) f.n_embd;
+ float g[64], u[64];
+ float* gp[1] = {g};
+ float* up[1] = {u};
+ for (int c0 = r0; c0 < r1; c0 += 64) {
+ const int c1 = std::min(r1, c0 + 64);
+ iq512x_rows(f.gu_type, blob + (size_t) c0 * f.gu_row, f.gu_row, n1, act, 1, gp, 0, c1 - c0);
+ iq512x_rows(f.gu_type, blob + f.up_off + (size_t) c0 * f.gu_row, f.gu_row, n1, act, 1, up, 0, c1 - c0);
+ for (int r = c0; r < c1; ++r) ff[0][r] = (g[r - c0] / (1.f + std::exp(-g[r - c0]))) * u[r - c0];
+ }
+ return;
+ }
const int mt_min = native_gu_mt_min(f.gu_type); // #152
// Unsloth UD-Q4_K_XL's Q4_K gate/up: the multi-token kernel is bit-exact against ggml's per-token dot (any group
// size, no #152 rule). Opt-in, STRATA_KQ256=1: measured no faster in the engine (a window's expert groups hold