Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
CPU expert rows: 512-bit grid lookups by gather where a gather is cheap; one IQ3 token on the exact 512-bit rows
Niko122116c4da9443fb
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 7dd95dfdf..41d9fd863 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1776,6 +1776,14 @@ if(STRATA_NATIVE_EXPERTS)# against ggml-cpu's dot, on random blocks; --bench: ms per expert on one thread, the weights streamed from DRAMadd_executable(iq_avx2_parity src/kernels/cpu/iq_avx2_parity.cpp)target_link_libraries(iq_avx2_parity PRIVATE strata_kernels_cpu)+ # the 512-bit i-quant rows against ggml-cpu's dot: the exact rows bit for bit at 1-4 tokens, the engine's multi-token rows to float+ # rounding, with the grid lookups as scalar loads and as gathers (SKIP without AVX-512)+ add_executable(iq512_gather_parity src/kernels/cpu/iq512_gather_parity.cpp)+ target_link_libraries(iq512_gather_parity PRIVATE strata_kernels_cpu)+ if(STRATA_BUILD_TESTS)+ add_test(NAME iq512_gather_parity_scalar COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ_GATHER=0 $<TARGET_FILE:iq512_gather_parity>)+ add_test(NAME iq512_gather_parity_gather COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ_GATHER=1 $<TARGET_FILE:iq512_gather_parity>)+ endif()# MODEL-UPDATE: the pool's expert rows per format on T threads against the RAM bandwidth (a benchmark, no test)add_executable(kq_cpu_bench src/kernels/cpu/kq_cpu_bench.cpp)target_link_libraries(kq_cpu_bench PRIVATE strata_kernels_cpu)diff --git a/include/strata/kernels/cpu/iq_avx512.hpp b/include/strata/kernels/cpu/iq_avx512.hppindex ae21dfd44..3f51b9dee 100644--- a/include/strata/kernels/cpu/iq_avx512.hpp+++ b/include/strata/kernels/cpu/iq_avx512.hpp@@ -8,6 +8,9 @@namespace strata::kernels::cpu {bool iq512_supported(int ggml_type) noexcept;+/// Whether the grid lookups of these rows are gathers (the default where cpu_gather_fast(), STRATA_IQ_GATHER=1 / 0 force it): the+/// same words, so the same bits as the scalar lookups, and 1.3-2x faster where a gather is cheap (Sapphire Rapids).+bool iq512_gather_on() noexcept;/// ff[t][r] = silu(gate_r . a[t]) * (up_r . a[t]), rows [r0, r1); gate rows at blob, up rows at blob + up_off.void iq512_gu_rows(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act,int nt, float* const* ff, int r0, int r1);diff --git a/src/kernels/cpu/iq512_gather_parity.cpp b/src/kernels/cpu/iq512_gather_parity.cppnew file mode 100644index 000000000..502880ea0--- /dev/null+++ b/src/kernels/cpu/iq512_gather_parity.cpp@@ -0,0 +1,76 @@+// iq512_gather_parity: the 512-bit i-quant rows (iq_avx512.cpp) against ggml-cpu's own dot.+// (a) iq512x_rows, for 1..4 tokens, is ggml-cpu's single-token vec_dot bit for bit (IQ3_XXS, IQ3_S, IQ2_S), whatever the+// grid lookup is (scalar loads or gathers: STRATA_IQ_GATHER=0 / 1, ctest runs both);+// (b) iq512_rows (the engine's multi-token rows) agrees with it to float rounding (another summation order).+// Synthetic blocks (random bytes, a fixed f16 scale), so no quantizer and no imatrix are needed. SKIP without AVX-512.+#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/iq_avx512.hpp"++#include "ggml.h"+#include "ggml-cpu.h"++#include <cmath>+#include <cstdio>+#include <cstring>+#include <vector>++namespace cpu = strata::kernels::cpu;++static int g_fail = 0;+#define CHECK(c) \+ do { \+ if (!(c)) { std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #c); ++g_fail; } \+ } while (0)++int main() {+ ggml_cpu_init();+ if (!cpu::cpu_avx512_ok()) {+ std::printf("iq512_gather_parity: SKIP (no AVX-512)\n");+ return 0;+ }+ std::printf("iq512_gather_parity: gather %d\n", (int) cpu::iq512_gather_on());+ constexpr int n = 2560, rows = 96, ntmax = 4;+ const ggml_type_traits_cpu* t8 = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K);+ uint32_t seed = 987654321u;+ auto rnd = [&seed] { seed = seed * 1664525u + 1013904223u; return seed >> 8; };+ const size_t ab = ggml_row_size(GGML_TYPE_Q8_K, n);+ std::vector<std::vector<uint8_t>> act(ntmax, std::vector<uint8_t>(ab));+ const void* ap[ntmax];+ for (int t = 0; t < ntmax; ++t) {+ std::vector<float> x(n);+ for (auto& v : x) v = ((float) (rnd() % 2001) - 1000.f) * 0.001f * (1.f + (float) t);+ t8->from_float(x.data(), act[t].data(), n);+ ap[t] = act[t].data();+ }+ for (int type : {18, 21, 22}) {+ const ggml_type_traits_cpu* tr = ggml_get_type_traits_cpu((ggml_type) type);+ CHECK(tr != nullptr && tr->vec_dot != nullptr);+ CHECK(cpu::iq512x_supported(type) && cpu::iq512_supported(type));+ const size_t rb = ggml_row_size((ggml_type) type, n), nb = (size_t) n / 256, bb = rb / nb;+ std::vector<uint8_t> w((size_t) rows * rb);+ for (auto& b : w) b = (uint8_t) rnd();+ for (int r = 0; r < rows; ++r)+ for (size_t k = 0; k < nb; ++k) { uint8_t* d = &w[(size_t) r * rb + k * bb]; d[0] = 0x66; d[1] = 0x2e; } // f16 ~0.1+ std::vector<float> want((size_t) ntmax * rows), gx((size_t) ntmax * rows), ge((size_t) ntmax * rows);+ for (int t = 0; t < ntmax; ++t)+ for (int r = 0; r < rows; ++r)+ tr->vec_dot(n, &want[(size_t) t * rows + r], 0, w.data() + (size_t) r * rb, 0, act[t].data(), 0, 1);+ for (int nt = 1; nt <= ntmax; ++nt) {+ float* ox[ntmax];+ float* oe[ntmax];+ for (int t = 0; t < ntmax; ++t) { ox[t] = &gx[(size_t) t * rows]; oe[t] = &ge[(size_t) t * rows]; }+ cpu::iq512x_rows(type, w.data(), rb, n, ap, nt, ox, 0, rows);+ cpu::iq512_rows(type, w.data(), rb, n, ap, nt, oe, 0, rows);+ for (int t = 0; t < nt; ++t)+ for (int r = 0; r < rows; ++r) {+ const float want_v = want[(size_t) t * rows + r];+ CHECK(std::memcmp(&gx[(size_t) t * rows + r], &want_v, 4) == 0); // (a) bit for bit+ const float e = ge[(size_t) t * rows + r];+ CHECK(std::fabs(e - want_v) <= 1e-3f * std::fabs(want_v) + 1e-3f); // (b) float-rounding agreement+ }+ }+ std::printf(" type %d: ok\n", type);+ }+ if (g_fail == 0) std::printf("iq512_gather_parity: OK\n");+ return g_fail == 0 ? 0 : 1;+}diff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cppindex 14330f6ec..aaa0adaa3 100644--- a/src/kernels/cpu/iq_avx512.cpp+++ b/src/kernels/cpu/iq_avx512.cpp@@ -9,6 +9,7 @@// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22). IQ1_M stays on ggml-cpu (no shipped// model has IQ1_M expert rows: the 'Coder IQ1_M' pack's gate/up are IQ2_S / IQ3_XXS / IQ3_S).#include "strata/kernels/cpu/iq_avx512.hpp"+#include "strata/kernels/cpu/expert_layout.hpp"#define GGML_COMMON_DECL_CPP#define GGML_COMMON_IMPL_CPP@@ -36,7 +37,14 @@ inline uint64_t u64(const uint8_t* p) { uint64_t v; std::memcpy(&v, p, 8); retur// E-2: the IQ3 grids by one AVX-512 gather (the same words, the same results) - measured 3-5% SLOWER than the scalar// lookups on a Ryzen 5 7600 (Zen 4 gathers are microcoded), so opt-in: STRATA_IQ_GATHER=1-const bool gather = std::getenv("STRATA_IQ_GATHER") != nullptr;+// 0.2.5: DEFAULT on where gathers are fast (Intel server and HEDT cores with AVX-512: cpu_gather_fast), off on Zen 4 (microcoded);+// STRATA_IQ_GATHER=1 / 0 force it either way. The same words are loaded, so the results are bit for bit the scalar decode's.+const bool gather = [] {+ const char* v = std::getenv("STRATA_IQ_GATHER");+ return v != nullptr && v[0] != '\0' ? std::atoi(v) != 0 : cpu_gather_fast();+}();+// the grid lookups by one 64-bit gather of eight entries (IQ2_XXS / IQ2_XS / IQ2_S): `idx` = eight 32-bit indices+inline __m512i gather64(__m256i idx, const void* grid) { return _mm512_i32gather_epi64(idx, grid, 8); }// E-2: software prefetch distance in bytes (STRATA_IQ_PREFETCH; 0 = off): -2-3% gate/up time on IQ3_S decode// (the rows are read at ~30 GB/s by 6 cores, near what DDR5 with 4 KB pages gives: the pool is memory-bound)const int prefetch_ahead = [] {@@ -62,7 +70,8 @@ template <> struct Fmt<16> { // IQ2_XXS: d, qs[32] u16static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {const uint8_t* q = b + 2 + 16 * j;const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);- g = _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],+ g = gather ? gather64(_mm256_cvtepu8_epi32(_mm_cvtsi64_si128((long long) ((uint64_t) a0 | ((uint64_t) b0 << 32)))), iq2xxs_grid)+ : _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],(long long) iq2xxs_grid[(b0 >> 8) & 255], (long long) iq2xxs_grid[b0 & 255],(long long) iq2xxs_grid[a0 >> 24], (long long) iq2xxs_grid[(a0 >> 16) & 255],(long long) iq2xxs_grid[(a0 >> 8) & 255], (long long) iq2xxs_grid[a0 & 255]);@@ -83,7 +92,8 @@ template <> struct Fmt<17> { // IQ2_XS: d, qs[32] u16 (9-bit grid index + 7-biconst uint8_t* q = b + 2 + 16 * j;uint16_t v[8];std::memcpy(v, q, 16);- g = _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],+ g = gather ? gather64(_mm256_and_si256(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*) q)), _mm256_set1_epi32(511)), iq2xs_grid)+ : _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],(long long) iq2xs_grid[v[5] & 511], (long long) iq2xs_grid[v[4] & 511],(long long) iq2xs_grid[v[3] & 511], (long long) iq2xs_grid[v[2] & 511],(long long) iq2xs_grid[v[1] & 511], (long long) iq2xs_grid[v[0] & 511]);@@ -101,6 +111,11 @@ template <> struct Fmt<22> { // IQ2_S: d, qs[64] (32 grid bytes, 32 sign bytesstatic inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {const uint8_t* qs = b + 2 + 8 * j;const uint8_t h0 = b[66 + 2 * j], h1 = b[66 + 2 * j + 1];+ if (gather) { // the 10-bit indices: the byte plus two high bits from qh (h0 for lanes 0-3, h1 for 4-7), one 64-bit gather+ const __m256i hi = _mm256_and_si256(_mm256_sllv_epi32(_mm256_setr_epi32(h0, h0, h0, h0, h1, h1, h1, h1),+ _mm256_setr_epi32(8, 6, 4, 2, 8, 6, 4, 2)), _mm256_set1_epi32(0x300));+ g = gather64(_mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) qs)), hi), iq2s_grid);+ } elseg = _mm512_set_epi64((long long) iq2s_grid[qs[7] | ((h1 << 2) & 0x300)], (long long) iq2s_grid[qs[6] | ((h1 << 4) & 0x300)],(long long) iq2s_grid[qs[5] | ((h1 << 6) & 0x300)], (long long) iq2s_grid[qs[4] | ((h1 << 8) & 0x300)],(long long) iq2s_grid[qs[3] | ((h0 << 2) & 0x300)], (long long) iq2s_grid[qs[2] | ((h0 << 4) & 0x300)],@@ -265,7 +280,8 @@ template <> struct FmtX<16> : Fmt<16> {static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {const uint8_t* q = b + 2 + 16 * j;const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);- g = _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],+ g = gather ? gather64(_mm256_cvtepu8_epi32(_mm_cvtsi64_si128((long long) ((uint64_t) a0 | ((uint64_t) b0 << 32)))), iq2xxs_grid)+ : _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],(long long) iq2xxs_grid[(b0 >> 8) & 255], (long long) iq2xxs_grid[b0 & 255],(long long) iq2xxs_grid[a0 >> 24], (long long) iq2xxs_grid[(a0 >> 16) & 255],(long long) iq2xxs_grid[(a0 >> 8) & 255], (long long) iq2xxs_grid[a0 & 255]);@@ -279,7 +295,8 @@ template <> struct FmtX<17> : Fmt<17> {const uint8_t* q = b + 2 + 16 * j;uint16_t v[8];std::memcpy(v, q, 16);- g = _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],+ g = gather ? gather64(_mm256_and_si256(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*) q)), _mm256_set1_epi32(511)), iq2xs_grid)+ : _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],(long long) iq2xs_grid[v[5] & 511], (long long) iq2xs_grid[v[4] & 511],(long long) iq2xs_grid[v[3] & 511], (long long) iq2xs_grid[v[2] & 511],(long long) iq2xs_grid[v[1] & 511], (long long) iq2xs_grid[v[0] & 511]);@@ -291,7 +308,8 @@ template <> struct FmtX<17> : Fmt<17> {template <> struct FmtX<18> : Fmt<18> {static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {const uint8_t* q = b + 2 + 16 * j;- g = _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],+ g = gather ? _mm512_i32gather_epi32(_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*) q)), iq3xxs_grid, 4)+ : _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],(int) iq3xxs_grid[q[11]], (int) iq3xxs_grid[q[10]], (int) iq3xxs_grid[q[9]], (int) iq3xxs_grid[q[8]],(int) iq3xxs_grid[q[7]], (int) iq3xxs_grid[q[6]], (int) iq3xxs_grid[q[5]], (int) iq3xxs_grid[q[4]],(int) iq3xxs_grid[q[3]], (int) iq3xxs_grid[q[2]], (int) iq3xxs_grid[q[1]], (int) iq3xxs_grid[q[0]]);@@ -578,6 +596,8 @@ static bool has_vpopcnt() {return ok;}+bool iq512_gather_on() noexcept { return gather; }+bool iq512x_supported(int type) noexcept {return has_vpopcnt() && (type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 29);}diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cppindex 7c491832c..ec1a68704 100644--- a/src/kernels/cpu/native_expert.cpp+++ b/src/kernels/cpu/native_expert.cpp@@ -39,6 +39,40 @@ bool tq_rows_own(int type) {return type >= (int) GGML_TYPE_COUNT || traits(type) == nullptr || traits(type)->vec_dot == nullptr;}+// 0.2.5: the 512-bit exact rows (iq512x_rows) take one token of an IQ3_XXS / IQ3_S expert only if this build's ggml-cpu dot gives+// the same bits on synthetic blocks of that type (a ggml built for AVX-512 may carry another summation order): checked once a type.+bool x512_one_ok(int type) {+ static std::mutex mu;+ static std::map<int, bool> memo;+ std::lock_guard<std::mutex> lk(mu);+ if (const auto it = memo.find(type); it != memo.end()) return it->second;+ bool ok = false;+ const ggml_type_traits_cpu* tr = traits(type);+ const ggml_type_traits_cpu* t8 = traits(GGML_TYPE_Q8_K);+ if (tr != nullptr && tr->vec_dot != nullptr && t8 != nullptr && t8->from_float != nullptr && iq512x_supported(type)) {+ constexpr int n = 2560, rows = 24;+ const size_t rb = ggml_row_size((ggml_type) type, n), nb = (size_t) n / 256, bb = rb / nb;+ std::vector<uint8_t> w((size_t) rows * rb), act(ggml_row_size(GGML_TYPE_Q8_K, n));+ uint32_t seed = 12345u;+ auto rnd = [&seed] { seed = seed * 1664525u + 1013904223u; return seed >> 8; };+ for (auto& b : w) b = (uint8_t) rnd();+ for (int r = 0; r < rows; ++r)+ for (size_t k = 0; k < nb; ++k) { uint8_t* d = &w[(size_t) r * rb + k * bb]; d[0] = 0x66; d[1] = 0x2e; } // d = ~0.1 (f16)+ std::vector<float> x(n);+ for (auto& v : x) v = ((float) (rnd() % 2001) - 1000.f) * 0.001f;+ t8->from_float(x.data(), act.data(), n);+ std::vector<float> got((size_t) rows), want((size_t) rows);+ const void* ap[1] = {act.data()};+ float* op[1] = {got.data()};+ iq512x_rows(type, w.data(), rb, n, ap, 1, op, 0, rows);+ for (int r = 0; r < rows; ++r) tr->vec_dot(n, &want[(size_t) r], 0, w.data() + (size_t) r * rb, 0, act.data(), 0, 1);+ ok = std::memcmp(got.data(), want.data(), got.size() * sizeof(float)) == 0;+ if (!ok) std::fprintf(stderr, "strata: the 512-bit exact one-token rows of type %d differ from this ggml-cpu's dot: not used\n", type);+ }+ memo[type] = ok;+ return ok;+}+void init_once() {static std::once_flag once;std::call_once(once, [] { ggml_cpu_init(); });@@ -299,6 +333,25 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*iq256_gu_rows_exact_one(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act[0], ff[0], r0, r1);return;}+ // 0.2.5 (AVX-512 CPUs where a gather is cheap: Intel Sapphire Rapids and later): ONE token of an IQ3_XXS / IQ3_S expert on the+ // 512-bit exact rows, ggml-cpu's single-token AVX2 dot bit for bit with the grid words gathered: 1.3x ggml's dot on a Xeon 8468+ // (x512_bench), the same output. Where the multi-token rule (#152) already sends one token to the multi-token kernel+ // (STRATA_IQ_MT_MIN=1) that stays; STRATA_NO_IQ512X1=1 turns this off; the first use checks the bits against this ggml.+ static const bool x512_one = avx512 && iq512_gather_on() && std::getenv("STRATA_NO_IQ512X1") == nullptr;+ if (x512_one && nt == 1 && nt < native_gu_mt_min(f.gu_type) && f.tail_off == 0 && (f.gu_type == 18 || f.gu_type == 21) &&+ x512_one_ok(f.gu_type)) {+ const int n1 = (int) f.n_embd;+ float g[64], u[64];+ float* gp[1] = {g};+ float* up[1] = {u};+ for (int c0 = r0; c0 < r1; c0 += 64) {+ const int c1 = std::min(r1, c0 + 64);+ iq512x_rows(f.gu_type, blob + (size_t) c0 * f.gu_row, f.gu_row, n1, act, 1, gp, 0, c1 - c0);+ iq512x_rows(f.gu_type, blob + f.up_off + (size_t) c0 * f.gu_row, f.gu_row, n1, act, 1, up, 0, c1 - c0);+ for (int r = c0; r < c1; ++r) ff[0][r] = (g[r - c0] / (1.f + std::exp(-g[r - c0]))) * u[r - c0];+ }+ return;+ }const int mt_min = native_gu_mt_min(f.gu_type); // #152// Unsloth UD-Q4_K_XL's Q4_K gate/up: the multi-token kernel is bit-exact against ggml's per-token dot (any group// size, no #152 rule). Opt-in, STRATA_KQ256=1: measured no faster in the engine (a window's expert groups hold