FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(cpu-moe): serve GGUF experts on the cpu and hybrid backends through ggml-cpu (#704)
Xiaoze Fanc8559fdff68c
diff --git a/docs/cli.md b/docs/cli.mdindex 4918a317..69848f2e 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -89,7 +89,7 @@ See [models.md](models.md#moe-strategies) for what each strategy does.| `--moe-cache-split` | equal | How the pools share the slots: `equal` (in proportion to their layer counts), or `profile:<path>`: every pool keeps `top_k` slots and the rest go to the most used experts in the profile's per-layer expert counts. With `offload`, a pool smaller than the expert count lowers `--max-running-requests` to its slots / top_k || `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts || `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor |-| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, fp8_block, nvfp4, mxfp4) |+| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, fp8_block, nvfp4, mxfp4, GGUF on x86 CPUs) || `--moe-hybrid-max-fetch` | auto | With `hybrid`: max experts fetched over PCIe per layer per step; rest computed on CPU || `--moe-prefill-hit-d2d` | off | Prefill: copy cache-hit experts device-side, stream only misses (CUDA >= 13) || `--disable-moe-prefill-overlap` | overlap on | Disable the two-buffer prefill copy overlap |diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 980bb043..326c467d 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -1479,6 +1479,10 @@ def _cpu_moe_executor_viable(model_config) -> bool:return Falseexpert_quant = getattr(model_config, "expert_quant", "none")fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")+ if fmt == "gguf":+ from freetoken.kernel import ggml_cpu++ return fmt in _cpu_moe.weight_formats() and ggml_cpu.select() is not Nonereturn fmt == "mxfp4" or fmt in _cpu_moe.weight_formats()diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex 6236816e..c4887efb 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -46,8 +46,9 @@ std::unique_ptr<MoeFormat> make_bf16_format(const MoeParams& p, const FormatArgsstd::unique_ptr<MoeFormat> make_nvfp4_format(const MoeParams& p, const FormatArgs& a);std::unique_ptr<MoeFormat> make_mxfp4_format(const MoeParams& p, const FormatArgs& a);std::unique_ptr<MoeFormat> make_ds_fp4_format(const MoeParams& p, const FormatArgs& a);-std::unique_ptr<MoeFormat> make_q4_0_format(const MoeParams& p, const FormatArgs& a);std::unique_ptr<MoeFormat> make_fp8_block_format(const MoeParams& p, const FormatArgs& a);+std::unique_ptr<MoeFormat> make_gguf_format(const MoeParams& p, const FormatArgs& a);+void set_ggml_cpu(uintptr_t addr);namespace {@@ -94,8 +95,8 @@ inline MoeTask from_word(uint64_t w) {// Every weight format this module serves, keyed by the name cpu_executor.py passes; weight_formats() exports the names.const std::pair<const char*, std::unique_ptr<MoeFormat> (*)(const MoeParams&, const FormatArgs&)> kFormats[] = {- {"bf16", make_bf16_format}, {"nvfp4", make_nvfp4_format}, {"mxfp4_triton", make_mxfp4_format},- {"ds_fp4", make_ds_fp4_format}, {"q4_0", make_q4_0_format}, {"fp8_block", make_fp8_block_format},+ {"bf16", make_bf16_format}, {"nvfp4", make_nvfp4_format}, {"mxfp4_triton", make_mxfp4_format},+ {"ds_fp4", make_ds_fp4_format}, {"fp8_block", make_fp8_block_format}, {"gguf", make_gguf_format},};std::unique_ptr<MoeFormat> make_format(const std::string& name, const MoeParams& p, const FormatArgs& a) {@@ -573,6 +574,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {// (act_apply falls through to gelu_tanh); the probe turns a stale extension// into a loud rebuild instruction instead of wrong model outputs.m.def("max_generic_act_id", []() { return static_cast<int>(freetoken::cpu_moe::ACT_SWIGLU_CLAMP); });+ // The ggml-cpu variant the gguf format calls: the kernel table address that freetoken.kernel.ggml_cpu.select() returns.+ m.def("set_ggml_cpu", &freetoken::cpu_moe::set_ggml_cpu, py::arg("api"));m.def("weight_formats", []() {py::list names;for (const auto& f : freetoken::cpu_moe::kFormats) names.append(f.first);diff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/gguf.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/gguf.cppnew file mode 100644index 00000000..c1641af5--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/gguf.cpp@@ -0,0 +1,107 @@+#include <atomic>+#include <mutex>+#include <stdexcept>+#include <string>+#include <vector>++#include "../format.h"+#include "cpu_kernels.h"++namespace freetoken::cpu_moe {+namespace {++std::atomic<const ft_ggml_cpu_api*> g_ggml{nullptr};++const ft_ggml_cpu_api& ggml_api() {+ const ft_ggml_cpu_api* api = g_ggml.load();+ if (api == nullptr) throw std::logic_error("CPU MoE gguf experts need a ggml-cpu variant; call _cpu_moe.set_ggml_cpu first");+ return *api;+}++} // namespace++void set_ggml_cpu(uintptr_t addr) {+ const auto* api = reinterpret_cast<const ft_ggml_cpu_api*>(addr);+ if (api == nullptr || api->abi != FT_GGML_CPU_ABI) throw std::invalid_argument("set_ggml_cpu: not a ggml-cpu kernel table of this build");+ const ft_ggml_cpu_api* prev = nullptr;+ if (!g_ggml.compare_exchange_strong(prev, api) && prev != api)+ throw std::invalid_argument(std::string("set_ggml_cpu: this process already uses the ") + prev->name + " ggml-cpu variant");+}++namespace {++// Packed ggml rows of the layers whose experts have this gate_up type and this down type, dotted by ggml-cpu against the activation converted to each type's vec_dot input (q8_0, q8_1 or q8_K, or the float type itself).+class GgufFormat final : public MoeFormat {+ public:+ GgufFormat(const MoeParams& p, const FormatArgs& a)+ : MoeFormat(p), ggml_(ggml_api()), gate_up_(a.bank("gate_up")), down_(a.bank("down")) {+ static std::once_flag tables;+ std::call_once(tables, ggml_.init);+ gu_ = kernel(static_cast<int>(a.option("gate_up_type")), p.H, "gate_up");+ dn_ = kernel(static_cast<int>(a.option("down_type")), p.I, "down");+ xf_.resize(p.H);+ xq_.resize(static_cast<size_t>(p.max_tokens) * gu_.act_row_bytes);+ gq_.resize(static_cast<size_t>(p.max_tokens) * p.top_k * dn_.act_row_bytes);+ }++ std::string isa_name() const override { return std::string(ggml_.name) + "(ggml-cpu)"; }++ void prep_input(int tok, const bf16_t* x) override {+ for (int i = 0; i < p.H; ++i) xf_[i] = bf16_to_f32(x[i]);+ gu_.from_float(xf_.data(), xq_.data() + static_cast<size_t>(tok) * gu_.act_row_bytes, p.H);+ }++ bool preps_inter() const override { return true; }++ void prep_inter(int64_t r, bf16_t* g) override {+ thread_local std::vector<float> gf;+ gf.resize(p.I);+ for (int i = 0; i < p.I; ++i) gf[i] = bf16_to_f32(g[i]);+ dn_.from_float(gf.data(), gq_.data() + static_cast<size_t>(r) * dn_.act_row_bytes, p.I);+ }++ void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float w_in, bf16_t* g) const override {+ const uint8_t* w = static_cast<const uint8_t*>(tbl_at(gate_up_, layer)) + static_cast<size_t>(e) * 2 * p.I * gu_.row_bytes;+ const uint8_t* xq = xq_.data() + static_cast<size_t>(tok) * gu_.act_row_bytes;+ gated_rows(p, i0, i1, w_in, g, [&](int row) {+ float s;+ gu_.vec_dot(p.H, &s, 0, w + static_cast<size_t>(row) * gu_.row_bytes, 0, xq, 0, 1);+ return s;+ });+ }++ void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {+ const uint8_t* w = static_cast<const uint8_t*>(tbl_at(down_, layer));+ for (int h = h0; h < h1; ++h)+ for (int j = 0; j < n; ++j)+ dn_.vec_dot(p.I, &out[j * kRowBlock + h - h0], 0, w + (static_cast<size_t>(routes[j].e) * p.H + h) * dn_.row_bytes, 0,+ gq_.data() + static_cast<size_t>(routes[j].r) * dn_.act_row_bytes, 0, 1);+ }++ private:+ ft_ggml_row_kernel kernel(int type, int k, const char* role) const {+ ft_ggml_row_kernel out{};+ const int rc = ggml_.row_kernel(type, k, &out);+ const std::string name = ggml_.type_name(type);+ if (rc == FT_GGML_NO_KERNEL) throw std::invalid_argument("ggml-cpu has no dot product for " + name + " experts");+ if (rc != FT_GGML_OK)+ throw std::invalid_argument("CPU MoE " + name + " " + role + " rows of " + std::to_string(k) + " values are not whole blocks");+ return out;+ }++ const ft_ggml_cpu_api& ggml_;+ const uint64_t* gate_up_;+ const uint64_t* down_;+ ft_ggml_row_kernel gu_{}, dn_{};+ std::vector<float> xf_;+ // the activation in the vec_dot input type: x per token, g per route+ AlignedVector<uint8_t> xq_, gq_;+};++} // namespace++std::unique_ptr<MoeFormat> make_gguf_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<GgufFormat>(p, a);+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/q4_0.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/q4_0.cppdeleted file mode 100644index 600a194e..00000000--- a/python/freetoken/kernel/csrc/cpu_moe/kernels/q4_0.cpp+++ /dev/null@@ -1,216 +0,0 @@-#include <stdexcept>-#include <vector>--#include "../format.h"--namespace freetoken::cpu_moe {-namespace {--// -------------------------------- Q4_0 (W4A8) ---------------------------------// Native GGUF Q4_0 experts (gemma4 GGUF): per-32 block = fp16 scale d + 16 packed-// bytes; byte j holds element j in its low nibble and j+16 in its high nibble, so a-// block's storage order is [lo0..lo15, hi0..hi15] and w = (nibble - 8) * d. Matches-// the reference dequant (models/gguf/dequant.py) and the packed banks the GPU offload-// path streams.-//-// llama.cpp ggml_vec_dot_q4_0_q8_0: W4A8. The activation is pre-quantized to Q8_0-// (per-32-block int8 ``aq`` + fp32 scale ``asb``); each block unpacks its 16 bytes to-// 32 int8 weights in [-8,7] (bytes_from_nibbles_32: low nibbles -> elems 0..15, high-// -> 16..31) and runs an integer block dot -- VPDPBUSD (AVX-VNNI) or VPMADDUBSW+VPMADDWD-// (AVX2) with the ggml sign trick |w|*(sign(w)*a)=w*a, or a scalar int loop -- then-// scales the block sum by wd*xd in fp32. No fp weight dequant / shuffle chain. The GPU-// offload path (ggml_moe_a8_vec / MMVQ) is also W4A8, so cpu and hybrid stay close.-using q4dot_fn = float (*)(const uint8_t*, const int8_t*, const float*, int);--float q4_0_dot_i8_scalar(const uint8_t* w, const int8_t* aq, const float* asb, int K) {- float acc = 0.0f;- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const uint8_t* blk = w + (size_t)b * 18;- uint16_t dh;- std::memcpy(&dh, blk, sizeof(dh));- const uint8_t* q = blk + 2; // 16 nibble bytes- const int8_t* a = aq + (size_t)b * 32;- int isum = 0;- for (int j = 0; j < 16; ++j) {- isum += ((int)(q[j] & 0x0F) - 8) * (int)a[j]; // elem j- isum += ((int)(q[j] >> 4) - 8) * (int)a[16 + j]; // elem 16+j- }- acc += fp16_to_f32(dh) * asb[b] * (float)isum;- }- return acc;-}--#if CPU_MOE_X86-// fp16 block scale -> fp32 via HW F16C (single value in lane 0).-CPU_MOE_TARGET("f16c")-static inline float q4_scale(uint16_t h) {- return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128((int)h)));-}--// Unpack one Q4_0 block's 16 bytes -> 32 int8 weights in [-8,7] (elems 0..15 = low-// nibbles, 16..31 = high nibbles). ``eight`` = _mm256_set1_epi8(8).-CPU_MOE_TARGET("avx2")-static inline __m256i q4_unpack32(const uint8_t* blk, __m128i mask, __m256i eight) {- const __m128i qb = _mm_loadu_si128(reinterpret_cast<const __m128i*>(blk + 2));- const __m128i lo = _mm_and_si128(qb, mask);- const __m128i hi = _mm_and_si128(_mm_srli_epi16(qb, 4), mask);- return _mm256_sub_epi8(_mm256_set_m128i(hi, lo), eight);-}--// AVX2 W4A8 (llama.cpp non-VNNI mul_sum_i8_pairs): integer block dot via VPMADDUBSW +-// VPMADDWD (sign trick), scaled by wd*xd. |aw*sa| pair sums <= 8*127*2 < 32767 -> no-// int16 saturation. This is the fast path on AVX2 CPUs without AVX-VNNI (and the-// avx512-tier fallback, since the block dot is 256-bit either way).-CPU_MOE_TARGET("avx2,fma,f16c")-float q4_0_dot_i8_avx2(const uint8_t* w, const int8_t* aq, const float* asb, int K) {- const __m128i mask = _mm_set1_epi8(0x0F);- const __m256i eight = _mm256_set1_epi8(8);- const __m256i ones16 = _mm256_set1_epi16(1);- __m256 accF = _mm256_setzero_ps();- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const uint8_t* blk = w + (size_t)b * 18;- _mm_prefetch(reinterpret_cast<const char*>(blk) + 512, _MM_HINT_T0);- uint16_t dh;- std::memcpy(&dh, blk, sizeof(dh));- __m256i wq = q4_unpack32(blk, mask, eight);- __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(aq + (size_t)b * 32));- __m256i aw = _mm256_sign_epi8(wq, wq); // |wq| (unsigned operand)- __m256i sa = _mm256_sign_epi8(a, wq); // sign(wq) * a (signed operand)- __m256i d32 = _mm256_madd_epi16(_mm256_maddubs_epi16(aw, sa), ones16); // 8 int32- accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(d32), _mm256_set1_ps(q4_scale(dh) * asb[b]), accF);- }- return hsum256(accF);-}--// AVX-VNNI W4A8: one VPDPBUSD per block (the fast path on modern CPUs).-CPU_MOE_TARGET("avx2,avxvnni,fma,f16c")-float q4_0_dot_i8_vnni(const uint8_t* w, const int8_t* aq, const float* asb, int K) {- const __m128i mask = _mm_set1_epi8(0x0F);- const __m256i eight = _mm256_set1_epi8(8);- __m256 accF = _mm256_setzero_ps();- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const uint8_t* blk = w + (size_t)b * 18;- _mm_prefetch(reinterpret_cast<const char*>(blk) + 512, _MM_HINT_T0);- uint16_t dh;- std::memcpy(&dh, blk, sizeof(dh));- __m256i wq = q4_unpack32(blk, mask, eight);- __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(aq + (size_t)b * 32));- __m256i aw = _mm256_sign_epi8(wq, wq); // |wq| (unsigned operand)- __m256i sa = _mm256_sign_epi8(a, wq); // sign(wq) * a (signed operand)- __m256i di = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), aw, sa);- // All 32 elems of the block share wd*xd; distribute over di's 8 partial sums and- // reduce at the end (equivalent to scale * block_total).- accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(di), _mm256_set1_ps(q4_scale(dh) * asb[b]), accF);- }- return hsum256(accF);-}-#endif // CPU_MOE_X86--// All tiers are W4A8 (int8 activations pre-quantized to Q8_0). AVX-VNNI is orthogonal to-// the ISA tier (gated by cpu_has_avxvnni() / FREETOKEN_CPU_MOE_NO_VNNI), so it wins when-// present; otherwise the 256-bit VPMADDUBSW kernel covers both the avx2 and avx512 tiers.-q4dot_fn select_q4dot() {- const IsaTier t = pick_isa();-#if CPU_MOE_X86- if (cpu_has_avxvnni()) return q4_0_dot_i8_vnni;- if (t >= ISA_AVX2) return q4_0_dot_i8_avx2;-#endif- (void)t;- return q4_0_dot_i8_scalar;-}--// Quantize a bf16 activation row to Q8_0 (llama.cpp): per-32-block symmetric int8 in-// natural order + one fp32 scale (amax/127) per block. Done once per token/route,-// amortized over every output row that the q4_0 W4A8 GEMV reads. K % 32 == 0.-void quant_q8_0(const bf16_t* x, int K, int8_t* aq, float* asb) {- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const bf16_t* xb = x + (size_t)b * 32;- float xf[32], amax = 0.0f;- for (int j = 0; j < 32; ++j) {- xf[j] = bf16_to_f32(xb[j]);- amax = std::max(amax, std::fabs(xf[j]));- }- const float d = amax > 0.0f ? amax / 127.0f : 1.0f;- asb[b] = d;- const float inv = amax > 0.0f ? 1.0f / d : 0.0f;- int8_t* o = aq + (size_t)b * 32;- for (int j = 0; j < 32; ++j)- o[j] = (int8_t)std::max(-127, std::min(127, (int)std::lround(xf[j] * inv)));- }-}--class Q4_0Format final : public MoeFormat {- public:- Q4_0Format(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), dot_(select_q4dot()) {- gate_up_ = a.bank("gate_up");- down_ = a.bank("down");- const size_t tokens = p.max_tokens, routes = (size_t)p.max_tokens * p.top_k;- xi8_.resize(tokens * p.H);- xas_.resize(tokens * (p.H / 32));- gi8_.resize(routes * p.I);- gas_.resize(routes * (p.I / 32));- if (p.H % 32 != 0 || p.I % 32 != 0)- throw std::runtime_error("Q4_0 CPU MoE requires H and I to be multiples of 32");- gu_row_bytes_ = (p.H / 32) * 18; // K = H (gate_up rows)- dn_row_bytes_ = (p.I / 32) * 18; // K = I (down rows)- // select_q4dot picks VPDPBUSD / VPMADDUBSW / scalar for the tier, so the tag reflects which of those it resolved to.- isa_ = std::string(tier_name(pick_isa())) + (cpu_has_avxvnni() ? "+vnni(q4_0-w4a8)" : "+q4_0-w4a8");- }-- std::string isa_name() const override { return isa_; }--- void prep_input(int tok, const bf16_t* x) override {- quant_q8_0(x, p.H, xi8_.data() + (size_t)tok * p.H, xas_.data() + (size_t)tok * (p.H / 32));- }-- bool preps_inter() const override { return true; }-- void prep_inter(int64_t r, bf16_t* g) override {- quant_q8_0(g, p.I, gi8_.data() + (size_t)r * p.I, gas_.data() + (size_t)r * (p.I / 32));- }-- void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float w_in,- bf16_t* g) const override {- const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));- const int8_t* xi8 = xi8_.data() + (size_t)tok * p.H;- const float* xas = xas_.data() + (size_t)tok * (p.H / 32);- gated_rows(p, i0, i1, w_in, g, [&](int row) {- return dot_(packed + ((size_t)e * (2 * p.I) + row) * (size_t)gu_row_bytes_, xi8, xas, p.H);- });- }-- void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {- const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(down_, layer));- for (int h = h0; h < h1; ++h) {- for (int j = 0; j < n; ++j) {- const size_t r = (size_t)routes[j].r;- out[j * kRowBlock + h - h0] = dot_(packed + ((size_t)routes[j].e * p.H + h) * (size_t)dn_row_bytes_,- gi8_.data() + r * p.I, gas_.data() + r * (p.I / 32), p.I);- }- }- }-- private:- q4dot_fn dot_;- // GGUF Q4_0 rows of 32-element blocks (fp16 scale + 16 nibble bytes): gate_up [E,2I,H/32*18], down [E,H,I/32*18].- const uint64_t* gate_up_ = nullptr;- const uint64_t* down_ = nullptr;- int gu_row_bytes_ = 0, dn_row_bytes_ = 0;- std::string isa_;- // Q8_0 activations: per-32-block int8 + fp32 scale, for the input and the intermediate.- AlignedVector<int8_t> xi8_, gi8_;- AlignedVector<float> xas_, gas_;-};--} // namespace--std::unique_ptr<MoeFormat> make_q4_0_format(const MoeParams& p, const FormatArgs& a) {- return std::make_unique<Q4_0Format>(p, a);-}--} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/ggml/VERSION b/python/freetoken/kernel/csrc/ggml/VERSIONindex 6eabb0c1..3c66f9b7 100644--- a/python/freetoken/kernel/csrc/ggml/VERSION+++ b/python/freetoken/kernel/csrc/ggml/VERSION@@ -1,7 +1,8 @@llama.cpp 8c2134a8755d8a9792af513dab20282affe903e4 (2026-10-14)-Only the ggml files that the GGUF ops (freetoken/ggml_ops.cu, built by kernel/ggml.py) compile or include:-ggml/include -> include, ggml/src -> src, ggml/src/ggml-cuda -> ggml-cuda.+Only the ggml files that FreeToken compiles or includes:+- the GGUF ops (freetoken/ggml_ops.cu, built by kernel/ggml.py): ggml/include -> include, ggml/src -> src, ggml/src/ggml-cuda -> ggml-cuda.+- the CPU MoE row kernels (freetoken/cpu_kernels.c and freetoken/variant_module.cpp, one freetoken.kernel.ggml_cpu module per x86 variant, built by setup.py): ggml/src/{ggml-quants.c,ggml-quants.h,ggml-backend-impl.h} -> src, ggml/include/ggml-cpu.h -> include, ggml/src/ggml-cpu/{quants.c,quants.h,vec.cpp,vec.h,ggml-cpu-impl.h,simd-mappings.h,arch-fallback.h,arch/x86/quants.c,arch/x86/cpu-feats.cpp} -> ggml-cpu. These have no local changes.Local changes, kept in freetoken.patch (already applied):- quantize.cu/.cuh: the q8_1 quantize kernels read fp32, bf16 or fp16 activations.diff --git a/python/freetoken/kernel/csrc/ggml/freetoken/cpu_kernels.c b/python/freetoken/kernel/csrc/ggml/freetoken/cpu_kernels.cnew file mode 100644index 00000000..79a03638--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/freetoken/cpu_kernels.c@@ -0,0 +1,172 @@+// The ggml-base symbols and lookup tables that the vendored ggml-cpu quant kernels reference, and the row kernel of each weight type.++#include "cpu_kernels.h"++#include "ggml-impl.h"+#include "ggml-cpu/ggml-cpu-impl.h"+#include "ggml-cpu/quants.h"+#include "ggml-cpu/simd-mappings.h"+#include "ggml-cpu/vec.h"++#include <stdarg.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++float ggml_table_f32_f16[1 << 16];+float ggml_table_f32_e8m0_half[1 << 8];+float ggml_table_f32_ue4m3[1 << 8];++void ggml_abort(const char * file, int line, const char * fmt, ...) {+ va_list args;+ va_start(args, fmt);+ fprintf(stderr, "ggml: %s:%d: ", file, line);+ vfprintf(stderr, fmt, args);+ va_end(args);+ fputc('\n', stderr);+ fflush(stderr);+ abort();+}++struct type_info {+ const char * name;+ int64_t blck_size;+ size_t type_size;+};++static const struct type_info k_types[GGML_TYPE_COUNT] = {+ [GGML_TYPE_F32] = {"f32", 1, sizeof(float)},+ [GGML_TYPE_F16] = {"f16", 1, sizeof(ggml_fp16_t)},+ [GGML_TYPE_BF16] = {"bf16", 1, sizeof(ggml_bf16_t)},+ [GGML_TYPE_Q1_0] = {"q1_0", QK1_0, sizeof(block_q1_0)},+ [GGML_TYPE_Q2_0] = {"q2_0", QK2_0, sizeof(block_q2_0)},+ [GGML_TYPE_Q4_0] = {"q4_0", QK4_0, sizeof(block_q4_0)},+ [GGML_TYPE_Q4_1] = {"q4_1", QK4_1, sizeof(block_q4_1)},+ [GGML_TYPE_Q5_0] = {"q5_0", QK5_0, sizeof(block_q5_0)},+ [GGML_TYPE_Q5_1] = {"q5_1", QK5_1, sizeof(block_q5_1)},+ [GGML_TYPE_Q8_0] = {"q8_0", QK8_0, sizeof(block_q8_0)},+ [GGML_TYPE_Q8_1] = {"q8_1", QK8_1, sizeof(block_q8_1)},+ [GGML_TYPE_MXFP4] = {"mxfp4", QK_MXFP4, sizeof(block_mxfp4)},+ [GGML_TYPE_NVFP4] = {"nvfp4", QK_NVFP4, sizeof(block_nvfp4)},+ [GGML_TYPE_Q2_K] = {"q2_K", QK_K, sizeof(block_q2_K)},+ [GGML_TYPE_Q3_K] = {"q3_K", QK_K, sizeof(block_q3_K)},+ [GGML_TYPE_Q4_K] = {"q4_K", QK_K, sizeof(block_q4_K)},+ [GGML_TYPE_Q5_K] = {"q5_K", QK_K, sizeof(block_q5_K)},+ [GGML_TYPE_Q6_K] = {"q6_K", QK_K, sizeof(block_q6_K)},+ [GGML_TYPE_Q8_K] = {"q8_K", QK_K, sizeof(block_q8_K)},+ [GGML_TYPE_TQ1_0] = {"tq1_0", QK_K, sizeof(block_tq1_0)},+ [GGML_TYPE_TQ2_0] = {"tq2_0", QK_K, sizeof(block_tq2_0)},+ [GGML_TYPE_IQ2_XXS] = {"iq2_xxs", QK_K, sizeof(block_iq2_xxs)},+ [GGML_TYPE_IQ2_XS] = {"iq2_xs", QK_K, sizeof(block_iq2_xs)},+ [GGML_TYPE_IQ2_S] = {"iq2_s", QK_K, sizeof(block_iq2_s)},+ [GGML_TYPE_IQ3_XXS] = {"iq3_xxs", QK_K, sizeof(block_iq3_xxs)},+ [GGML_TYPE_IQ3_S] = {"iq3_s", QK_K, sizeof(block_iq3_s)},+ [GGML_TYPE_IQ1_S] = {"iq1_s", QK_K, sizeof(block_iq1_s)},+ [GGML_TYPE_IQ1_M] = {"iq1_m", QK_K, sizeof(block_iq1_m)},+ [GGML_TYPE_IQ4_NL] = {"iq4_nl", QK4_NL, sizeof(block_iq4_nl)},+ [GGML_TYPE_IQ4_XS] = {"iq4_xs", QK_K, sizeof(block_iq4_xs)},+};++static int known_type(int type) { return type >= 0 && type < GGML_TYPE_COUNT && k_types[type].name != NULL; }++int64_t ggml_blck_size(enum ggml_type type) { return k_types[type].blck_size; }++size_t ggml_type_size(enum ggml_type type) { return k_types[type].type_size; }++size_t ggml_row_size(enum ggml_type type, int64_t ne) { return k_types[type].type_size * ne / k_types[type].blck_size; }++const char * ggml_type_name(enum ggml_type type) { return known_type(type) ? k_types[type].name : "unknown"; }++const char * ft_ggml_type_name(int type) { return ggml_type_name((enum ggml_type) type); }++struct dot_info {+ ft_ggml_vec_dot_t vec_dot;+ int act_type;+};++// The weight types of type_traits_cpu in ggml-cpu.c, with their vec_dot and vec_dot_type.+static const struct dot_info k_dots[GGML_TYPE_COUNT] = {+ [GGML_TYPE_F32] = {(ft_ggml_vec_dot_t) ggml_vec_dot_f32, GGML_TYPE_F32},+ [GGML_TYPE_F16] = {(ft_ggml_vec_dot_t) ggml_vec_dot_f16, GGML_TYPE_F16},+ [GGML_TYPE_BF16] = {(ft_ggml_vec_dot_t) ggml_vec_dot_bf16, GGML_TYPE_BF16},+ [GGML_TYPE_Q1_0] = {ggml_vec_dot_q1_0_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_Q2_0] = {ggml_vec_dot_q2_0_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_Q4_0] = {ggml_vec_dot_q4_0_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_Q4_1] = {ggml_vec_dot_q4_1_q8_1, GGML_TYPE_Q8_1},+ [GGML_TYPE_Q5_0] = {ggml_vec_dot_q5_0_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_Q5_1] = {ggml_vec_dot_q5_1_q8_1, GGML_TYPE_Q8_1},+ [GGML_TYPE_Q8_0] = {ggml_vec_dot_q8_0_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_MXFP4] = {ggml_vec_dot_mxfp4_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_NVFP4] = {ggml_vec_dot_nvfp4_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_Q2_K] = {ggml_vec_dot_q2_K_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_Q3_K] = {ggml_vec_dot_q3_K_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_Q4_K] = {ggml_vec_dot_q4_K_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_Q5_K] = {ggml_vec_dot_q5_K_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_Q6_K] = {ggml_vec_dot_q6_K_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_TQ1_0] = {ggml_vec_dot_tq1_0_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_TQ2_0] = {ggml_vec_dot_tq2_0_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ2_XXS] = {ggml_vec_dot_iq2_xxs_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ2_XS] = {ggml_vec_dot_iq2_xs_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ2_S] = {ggml_vec_dot_iq2_s_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ3_XXS] = {ggml_vec_dot_iq3_xxs_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ3_S] = {ggml_vec_dot_iq3_s_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ1_S] = {ggml_vec_dot_iq1_s_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ1_M] = {ggml_vec_dot_iq1_m_q8_K, GGML_TYPE_Q8_K},+ [GGML_TYPE_IQ4_NL] = {ggml_vec_dot_iq4_nl_q8_0, GGML_TYPE_Q8_0},+ [GGML_TYPE_IQ4_XS] = {ggml_vec_dot_iq4_xs_q8_K, GGML_TYPE_Q8_K},+};++static void fp32_to_fp32_row(const float * x, void * y, int64_t k) { memcpy(y, x, (size_t) k * sizeof(float)); }++static void fp32_to_fp16_row(const float * x, void * y, int64_t k) {+ ggml_fp16_t * out = y;+ for (int64_t i = 0; i < k; ++i) {+ out[i] = GGML_CPU_FP32_TO_FP16(x[i]);+ }+}++static void fp32_to_bf16_row(const float * x, void * y, int64_t k) {+ ggml_bf16_t * out = y;+ for (int64_t i = 0; i < k; ++i) {+ out[i] = GGML_FP32_TO_BF16(x[i]);+ }+}++static ft_ggml_from_float_t act_quantizer(int act_type) {+ switch (act_type) {+ case GGML_TYPE_F32: return fp32_to_fp32_row;+ case GGML_TYPE_F16: return fp32_to_fp16_row;+ case GGML_TYPE_BF16: return fp32_to_bf16_row;+ case GGML_TYPE_Q8_0: return quantize_row_q8_0;+ case GGML_TYPE_Q8_1: return quantize_row_q8_1;+ case GGML_TYPE_Q8_K: return quantize_row_q8_K;+ default: return NULL;+ }+}++// The tables ggml_cpu_init fills for the quant kernels.+void ft_ggml_cpu_init(void) {+ for (int i = 0; i < (1 << 16); ++i) {+ ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32((ggml_fp16_t) i);+ }+ for (int i = 0; i < (1 << 8); ++i) {+ ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i);+ ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32((uint8_t) i);+ }+}++int ft_ggml_cpu_row_kernel(int type, int64_t k, struct ft_ggml_row_kernel * out) {+ if (!known_type(type) || k_dots[type].vec_dot == NULL) {+ return FT_GGML_NO_KERNEL;+ }+ const int act = k_dots[type].act_type;+ if (k % k_types[type].blck_size != 0 || k % k_types[act].blck_size != 0) {+ return FT_GGML_PARTIAL_BLOCK;+ }+ out->vec_dot = k_dots[type].vec_dot;+ out->from_float = act_quantizer(act);+ out->act_type = act;+ out->row_bytes = ggml_row_size((enum ggml_type) type, k);+ out->act_row_bytes = ggml_row_size((enum ggml_type) act, k);+ return FT_GGML_OK;+}diff --git a/python/freetoken/kernel/csrc/ggml/freetoken/cpu_kernels.h b/python/freetoken/kernel/csrc/ggml/freetoken/cpu_kernels.hnew file mode 100644index 00000000..d58c1748--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/freetoken/cpu_kernels.h@@ -0,0 +1,43 @@+// ggml-cpu row kernels for the CPU MoE executor. setup.py builds them once per x86 variant into freetoken.kernel.ggml_cpu._<variant>; call them only through the table of a variant whose score() is not 0.+#pragma once++#include <stddef.h>+#include <stdint.h>++#ifdef __cplusplus+extern "C" {+#endif++typedef void (*ft_ggml_vec_dot_t)(int n, float * s, size_t bs, const void * x, size_t bx, const void * y, size_t by, int nrc);+typedef void (*ft_ggml_from_float_t)(const float * x, void * y, int64_t k);++struct ft_ggml_row_kernel {+ ft_ggml_vec_dot_t vec_dot; // one weight row . one activation row in act_type+ ft_ggml_from_float_t from_float; // fp32 activation row -> act_type+ int act_type;+ size_t row_bytes; // one weight row of k values+ size_t act_row_bytes; // one activation row of k values in act_type+};++enum { FT_GGML_OK = 0, FT_GGML_NO_KERNEL = 1, FT_GGML_PARTIAL_BLOCK = 2 };++void ft_ggml_cpu_init(void);++// The kernels for weight rows of ggml type `type` over k values.+int ft_ggml_cpu_row_kernel(int type, int64_t k, struct ft_ggml_row_kernel * out);++const char * ft_ggml_type_name(int type);++// One variant's functions, as its module's api() hands them to _cpu_moe.set_ggml_cpu.+#define FT_GGML_CPU_ABI 1+struct ft_ggml_cpu_api {+ int abi; // FT_GGML_CPU_ABI of the module that built it+ const char * name; // the variant, e.g. "haswell"+ void (*init)(void);+ int (*row_kernel)(int type, int64_t k, struct ft_ggml_row_kernel * out);+ const char * (*type_name)(int type);+};++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/freetoken/variant_module.cpp b/python/freetoken/kernel/csrc/ggml/freetoken/variant_module.cppnew file mode 100644index 00000000..8f3d3e52--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/freetoken/variant_module.cpp@@ -0,0 +1,32 @@+// One ggml-cpu variant as a Python module: score() is 0 unless this CPU has every ISA extension the variant is built for, and api() is the address of its kernel table.+// setup.py builds this file without the variant's ISA flags, like upstream's <variant>-feats target, so importing the module and calling score() is safe on any x86 CPU.+#define PY_SSIZE_T_CLEAN+#include <Python.h>++#include "cpu_kernels.h"+#include "ggml-cpu/arch/x86/cpu-feats.cpp"++#define FT_CAT_(a, b) a##b+#define FT_CAT(a, b) FT_CAT_(a, b)+#define FT_STR_(a) #a+#define FT_STR(a) FT_STR_(a)++namespace {++const ft_ggml_cpu_api kApi = {FT_GGML_CPU_ABI, FT_STR(FT_GGML_VARIANT), ft_ggml_cpu_init, ft_ggml_cpu_row_kernel, ft_ggml_type_name};++PyObject* score(PyObject*, PyObject*) { return PyLong_FromLong(ggml_backend_cpu_x86_score()); }++PyObject* api(PyObject*, PyObject*) { return PyLong_FromVoidPtr(const_cast<ft_ggml_cpu_api*>(&kApi)); }++PyMethodDef kMethods[] = {+ {"score", score, METH_NOARGS, "0 if this CPU lacks an ISA extension the variant is built for, else upstream's variant score."},+ {"api", api, METH_NOARGS, "The address of the variant's kernel table, for _cpu_moe.set_ggml_cpu."},+ {nullptr, nullptr, 0, nullptr},+};++PyModuleDef kModule = {PyModuleDef_HEAD_INIT, "_" FT_STR(FT_GGML_VARIANT), nullptr, -1, kMethods};++} // namespace++PyMODINIT_FUNC FT_CAT(PyInit__, FT_GGML_VARIANT)(void) { return PyModule_Create(&kModule); }diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch-fallback.h b/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch-fallback.hnew file mode 100644index 00000000..4e59d0ae--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch-fallback.h@@ -0,0 +1,375 @@++#pragma once++// Rename `_generic` functions if no native implementation is available.+// This effectively selects the generic implementation.++#if defined(GGML_CPU_GENERIC)+// quants.c+#define quantize_row_q8_0_generic quantize_row_q8_0+#define quantize_row_q8_1_generic quantize_row_q8_1+#define quantize_row_q8_K_generic quantize_row_q8_K+#define ggml_vec_dot_q4_0_q8_0_generic ggml_vec_dot_q4_0_q8_0+#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1+#define ggml_vec_dot_q5_0_q8_0_generic ggml_vec_dot_q5_0_q8_0+#define ggml_vec_dot_q5_1_q8_1_generic ggml_vec_dot_q5_1_q8_1+#define ggml_vec_dot_q8_0_q8_0_generic ggml_vec_dot_q8_0_q8_0+#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0+#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0+#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K+#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K+#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K+#define ggml_vec_dot_q3_K_q8_K_generic ggml_vec_dot_q3_K_q8_K+#define ggml_vec_dot_q4_K_q8_K_generic ggml_vec_dot_q4_K_q8_K+#define ggml_vec_dot_q5_K_q8_K_generic ggml_vec_dot_q5_K_q8_K+#define ggml_vec_dot_q6_K_q8_K_generic ggml_vec_dot_q6_K_q8_K+#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K+#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K+#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K+#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K+#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K+#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K+#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K+#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0+#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K+// repack.cpp+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4+#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0+#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0+#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0+#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC)+// repack.cpp+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K+#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)+// repack.cpp+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0+#elif defined(__POWERPC__) || defined(__powerpc__)+// ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679+// quants.c+#define quantize_row_q8_K_generic quantize_row_q8_K+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0+#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0+#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K+#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K+#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K+// repack.cpp+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4+#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0+#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0+#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0+#elif defined(__loongarch64)+// quants.c+#define quantize_row_q8_K_generic quantize_row_q8_K+#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K+#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K+#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K+#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0+#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0+// repack.cpp+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4+#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0+#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0+#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0+#elif defined(__riscv)+// quants.c+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0+// repack.cpp+#define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4+#define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0+#elif defined(__s390x__)+// quants.c+#define quantize_row_q8_K_generic quantize_row_q8_K+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0+#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K+#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K+#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K+#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K+#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K+#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K+#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K+#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K+#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K+#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K+// repack.cpp+#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0+#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0+#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0+#elif defined(__wasm__)+// quants.c+#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K+#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K+#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K+#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K+#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K+#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K+#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K+#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K+#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K+#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0+#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K+#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0+#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0+// repack.cpp+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4+#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0+#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0+#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch/x86/cpu-feats.cpp b/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch/x86/cpu-feats.cppnew file mode 100644index 00000000..d775a036--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch/x86/cpu-feats.cpp@@ -0,0 +1,327 @@+#include "ggml-backend-impl.h"++#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))++#ifdef _MSC_VER+#include <intrin.h>+#endif++#include <cstring>+#include <vector>+#include <bitset>+#include <array>+#include <string>++// ref: https://cdrdv2-public.intel.com/782156/325383-sdm-vol-2abcd.pdf+struct cpuid_x86 {+ bool SSE3(void) { return f_1_ecx[0]; }+ bool PCLMULQDQ(void) { return f_1_ecx[1]; }+ bool MONITOR(void) { return f_1_ecx[3]; }+ bool SSSE3(void) { return f_1_ecx[9]; }+ bool FMA(void) { return f_1_ecx[12]; }+ bool CMPXCHG16B(void) { return f_1_ecx[13]; }+ bool SSE41(void) { return f_1_ecx[19]; }+ bool SSE42(void) { return f_1_ecx[20]; }+ bool MOVBE(void) { return f_1_ecx[22]; }+ bool POPCNT(void) { return f_1_ecx[23]; }+ bool AES(void) { return f_1_ecx[25]; }+ bool XSAVE(void) { return f_1_ecx[26]; }+ bool OSXSAVE(void) { return f_1_ecx[27]; }+ bool AVX(void) { return f_1_ecx[28]; }+ bool F16C(void) { return f_1_ecx[29]; }+ bool RDRAND(void) { return f_1_ecx[30]; }++ bool MSR(void) { return f_1_edx[5]; }+ bool CX8(void) { return f_1_edx[8]; }+ bool SEP(void) { return f_1_edx[11]; }+ bool CMOV(void) { return f_1_edx[15]; }+ bool CLFSH(void) { return f_1_edx[19]; }+ bool MMX(void) { return f_1_edx[23]; }+ bool FXSR(void) { return f_1_edx[24]; }+ bool SSE(void) { return f_1_edx[25]; }+ bool SSE2(void) { return f_1_edx[26]; }++ bool FSGSBASE(void) { return f_7_ebx[0]; }+ bool BMI1(void) { return f_7_ebx[3]; }+ bool HLE(void) { return is_intel && f_7_ebx[4]; }+ bool AVX2(void) { return f_7_ebx[5]; }+ bool BMI2(void) { return f_7_ebx[8]; }+ bool ERMS(void) { return f_7_ebx[9]; }+ bool INVPCID(void) { return f_7_ebx[10]; }+ bool RTM(void) { return is_intel && f_7_ebx[11]; }+ bool AVX512F(void) { return f_7_ebx[16]; }+ bool AVX512DQ(void) { return f_7_ebx[17]; }+ bool RDSEED(void) { return f_7_ebx[18]; }+ bool ADX(void) { return f_7_ebx[19]; }+ bool AVX512PF(void) { return f_7_ebx[26]; }+ bool AVX512ER(void) { return f_7_ebx[27]; }+ bool AVX512CD(void) { return f_7_ebx[28]; }+ bool AVX512BW(void) { return f_7_ebx[30]; }+ bool AVX512VL(void) { return f_7_ebx[31]; }++ bool SHA(void) { return f_7_ebx[29]; }++ bool PREFETCHWT1(void) { return f_7_ecx[0]; }++ bool LAHF(void) { return f_81_ecx[0]; }+ bool LZCNT(void) { return is_intel && f_81_ecx[5]; }+ bool ABM(void) { return is_amd && f_81_ecx[5]; }+ bool SSE4a(void) { return is_amd && f_81_ecx[6]; }+ bool XOP(void) { return is_amd && f_81_ecx[11]; }+ bool TBM(void) { return is_amd && f_81_ecx[21]; }++ bool SYSCALL(void) { return is_intel && f_81_edx[11]; }+ bool MMXEXT(void) { return is_amd && f_81_edx[22]; }+ bool RDTSCP(void) { return is_intel && f_81_edx[27]; }+ bool _3DNOWEXT(void) { return is_amd && f_81_edx[30]; }+ bool _3DNOW(void) { return is_amd && f_81_edx[31]; }++ bool AVX512_VBMI(void) { return f_7_ecx[1]; }+ bool AVX512_VNNI(void) { return f_7_ecx[11]; }+ bool AVX512_FP16(void) { return f_7_edx[23]; }+ bool AVX512_BF16(void) { return f_7_1_eax[5]; }+ bool AVX_VNNI(void) { return f_7_1_eax[4]; }++ bool AMX_TILE(void) { return f_7_edx[24]; }+ bool AMX_INT8(void) { return f_7_edx[25]; }+ bool AMX_FP16(void) { return f_7_1_eax[21]; }+ bool AMX_BF16(void) { return f_7_edx[22]; }++#ifdef _MSC_VER+ static void cpuid(int cpu_info[4], int eax) {+ __cpuid(cpu_info, eax);+ }+ static void cpuidex(int cpu_info[4], int eax, int ecx) {+ __cpuidex(cpu_info, eax, ecx);+ }+#else+ static void cpuid(int cpu_info[4], int eax) {+ __asm__ __volatile__(+ "cpuid"+ : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])+ : "a"(eax), "c"(0));+ }+ static void cpuidex(int cpu_info[4], int eax, int ecx) {+ __asm__ __volatile__(+ "cpuid"+ : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])+ : "a"(eax), "c"(ecx));+ }+#endif++ cpuid_x86() {+ std::array<int, 4> cpui;+ std::vector<std::array<int, 4>> data;++ // calling __cpuid with 0x0 as the function_id argument+ // gets the number of the highest valid function ID.+ cpuid(cpui.data(), 0);+ int n_ids = cpui[0];++ for (int i = 0; i <= n_ids; ++i) {+ cpuidex(cpui.data(), i, 0);+ data.push_back(cpui);+ }++ // capture vendor string+ char vendor[0x20] = {};+ *reinterpret_cast<int *>(vendor) = data[0][1];+ *reinterpret_cast<int *>(vendor + 4) = data[0][3];+ *reinterpret_cast<int *>(vendor + 8) = data[0][2];+ this->vendor = vendor;+ if (this->vendor == "GenuineIntel") {+ is_intel = true;+ } else if (this->vendor == "AuthenticAMD") {+ is_amd = true;+ }++ // load bitset with flags for function 0x00000001+ if (n_ids >= 1) {+ f_1_ecx = data[1][2];+ f_1_edx = data[1][3];+ }++ // load bitset with flags for function 0x00000007+ if (n_ids >= 7) {+ f_7_ebx = data[7][1];+ f_7_ecx = data[7][2];+ f_7_edx = data[7][3];+ cpuidex(cpui.data(), 7, 1);+ f_7_1_eax = cpui[0];+ }++ // calling __cpuid with 0x80000000 as the function_id argument+ // gets the number of the highest valid extended ID.+ cpuid(cpui.data(), 0x80000000);+ unsigned int n_ex_ids = cpui[0];++ std::vector<std::array<int, 4>> ext_data;+ for (unsigned int i = 0x80000000; i <= n_ex_ids; ++i) {+ cpuidex(cpui.data(), i, 0);+ ext_data.push_back(cpui);+ }++ // load bitset with flags for function 0x80000001+ if (n_ex_ids >= 0x80000001) {+ f_81_ecx = ext_data[1][2];+ f_81_edx = ext_data[1][3];+ }++ // interpret CPU brand string if reported+ char brand[0x40] = {};+ if (n_ex_ids >= 0x80000004) {+ std::memcpy(brand, ext_data[2].data(), sizeof(cpui));+ std::memcpy(brand + 16, ext_data[3].data(), sizeof(cpui));+ std::memcpy(brand + 32, ext_data[4].data(), sizeof(cpui));+ this->brand = brand;+ }+ }++ bool is_intel = false;+ bool is_amd = false;+ std::string vendor;+ std::string brand;+ std::bitset<32> f_1_ecx;+ std::bitset<32> f_1_edx;+ std::bitset<32> f_7_ebx;+ std::bitset<32> f_7_ecx;+ std::bitset<32> f_7_edx;+ std::bitset<32> f_7_1_eax;+ std::bitset<32> f_81_ecx;+ std::bitset<32> f_81_edx;+};++#if 0+void test_x86_is() {+ cpuid_x86 is;+ printf("CPU Vendor: %s\n", is.vendor.c_str());+ printf("Brand: %s\n", is.brand.c_str());+ printf("is_intel: %d\n", is.is_intel);+ printf("is_amd: %d\n", is.is_amd);+ printf("sse3: %d\n", is.SSE3());+ printf("pclmulqdq: %d\n", is.PCLMULQDQ());+ printf("ssse3: %d\n", is.SSSE3());+ printf("fma: %d\n", is.FMA());+ printf("cmpxchg16b: %d\n", is.CMPXCHG16B());+ printf("sse41: %d\n", is.SSE41());+ printf("sse42: %d\n", is.SSE42());+ printf("movbe: %d\n", is.MOVBE());+ printf("popcnt: %d\n", is.POPCNT());+ printf("aes: %d\n", is.AES());+ printf("xsave: %d\n", is.XSAVE());+ printf("osxsave: %d\n", is.OSXSAVE());+ printf("avx: %d\n", is.AVX());+ printf("f16c: %d\n", is.F16C());+ printf("rdrand: %d\n", is.RDRAND());+ printf("msr: %d\n", is.MSR());+ printf("cx8: %d\n", is.CX8());+ printf("sep: %d\n", is.SEP());+ printf("cmov: %d\n", is.CMOV());+ printf("clflush: %d\n", is.CLFSH());+ printf("mmx: %d\n", is.MMX());+ printf("fxsr: %d\n", is.FXSR());+ printf("sse: %d\n", is.SSE());+ printf("sse2: %d\n", is.SSE2());+ printf("fsgsbase: %d\n", is.FSGSBASE());+ printf("bmi1: %d\n", is.BMI1());+ printf("hle: %d\n", is.HLE());+ printf("avx2: %d\n", is.AVX2());+ printf("bmi2: %d\n", is.BMI2());+ printf("erms: %d\n", is.ERMS());+ printf("invpcid: %d\n", is.INVPCID());+ printf("rtm: %d\n", is.RTM());+ printf("avx512f: %d\n", is.AVX512F());+ printf("rdseed: %d\n", is.RDSEED());+ printf("adx: %d\n", is.ADX());+ printf("avx512pf: %d\n", is.AVX512PF());+ printf("avx512er: %d\n", is.AVX512ER());+ printf("avx512cd: %d\n", is.AVX512CD());+ printf("sha: %d\n", is.SHA());+ printf("prefetchwt1: %d\n", is.PREFETCHWT1());+ printf("lahf: %d\n", is.LAHF());+ printf("lzcnt: %d\n", is.LZCNT());+ printf("abm: %d\n", is.ABM());+ printf("sse4a: %d\n", is.SSE4a());+ printf("xop: %d\n", is.XOP());+ printf("tbm: %d\n", is.TBM());+ printf("syscall: %d\n", is.SYSCALL());+ printf("mmxext: %d\n", is.MMXEXT());+ printf("rdtscp: %d\n", is.RDTSCP());+ printf("3dnowext: %d\n", is._3DNOWEXT());+ printf("3dnow: %d\n", is._3DNOW());+ printf("avx512_vbmi: %d\n", is.AVX512_VBMI());+ printf("avx512_vnni: %d\n", is.AVX512_VNNI());+ printf("avx512_fp16: %d\n", is.AVX512_FP16());+ printf("avx512_bf16: %d\n", is.AVX512_BF16());+ printf("amx_tile: %d\n", is.AMX_TILE());+ printf("amx_int8: %d\n", is.AMX_INT8());+ printf("amx_fp16: %d\n", is.AMX_FP16());+ printf("amx_bf16: %d\n", is.AMX_BF16());+}+#endif++static int ggml_backend_cpu_x86_score() {+ // FIXME: this does not check for OS support++ int score = 1;+ cpuid_x86 is;++#ifdef GGML_FMA+ if (!is.FMA()) { return 0; }+ score += 1;+#endif+#ifdef GGML_F16C+ if (!is.F16C()) { return 0; }+ score += 1<<1;+#endif+#ifdef GGML_SSE42+ if (!is.SSE42()) { return 0; }+ score += 1<<2;+#endif+#ifdef GGML_BMI2+ if (!is.BMI2()) { return 0; }+ score += 1<<3;+#endif+#ifdef GGML_AVX+ if (!is.AVX()) { return 0; }+ score += 1<<4;+#endif+#ifdef GGML_AVX2+ if (!is.AVX2()) { return 0; }+ score += 1<<5;+#endif+#ifdef GGML_AVX_VNNI+ if (!is.AVX_VNNI()) { return 0; }+ score += 1<<6;+#endif+#ifdef GGML_AVX512+ if (!is.AVX512F()) { return 0; }+ if (!is.AVX512CD()) { return 0; }+ if (!is.AVX512VL()) { return 0; }+ if (!is.AVX512DQ()) { return 0; }+ if (!is.AVX512BW()) { return 0; }+ score += 1<<7;+#endif+#ifdef GGML_AVX512_VBMI+ if (!is.AVX512_VBMI()) { return 0; }+ score += 1<<8;+#endif+#ifdef GGML_AVX512_BF16+ if (!is.AVX512_BF16()) { return 0; }+ score += 1<<9;+#endif+#ifdef GGML_AVX512_VNNI+ if (!is.AVX512_VNNI()) { return 0; }+ score += 1<<10;+#endif+#ifdef GGML_AMX_INT8+ if (!is.AMX_INT8()) { return 0; }+ score += 1<<11;+#endif++ return score;+}++GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_x86_score)++#endif // defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch/x86/quants.c b/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch/x86/quants.cnew file mode 100644index 00000000..21cd6312--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/arch/x86/quants.c@@ -0,0 +1,4172 @@+#define GGML_COMMON_IMPL_C+#include "ggml-common.h"+#include "ggml-quants.h"+#include "ggml-impl.h"+#include "ggml-cpu.h"+#include "simd-mappings.h"++#include "../../quants.h"+#include "../../ggml-cpu-impl.h"++#include <math.h>+#include <string.h>+#include <assert.h>+#include <stdlib.h> // for qsort+#include <stdio.h> // for GGML_ASSERT++#define GROUP_MAX_EPS 1e-15f+#define GROUP_MAX_EPS_IQ3_XXS 1e-8f+#define GROUP_MAX_EPS_IQ2_S 1e-8f+#define GROUP_MAX_EPS_IQ1_M 1e-7f+#define GROUP_MAX_EPS_IQ1_S 1e-12f++#define UNUSED GGML_UNUSED++// some compilers don't provide _mm256_set_m128i, e.g. gcc 7+#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)++#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)+// multiply int8_t, add results pairwise twice+static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {+ // Get absolute values of x vectors+ const __m128i ax = _mm_sign_epi8(x, x);+ // Sign the values of the y vectors+ const __m128i sy = _mm_sign_epi8(y, x);+ // Perform multiplication and create 16-bit values+ const __m128i dot = _mm_maddubs_epi16(ax, sy);+ const __m128i ones = _mm_set1_epi16(1);+ return _mm_madd_epi16(ones, dot);+}++#if __AVX__ || __AVX2__ || __AVX512F__+// horizontally add 8 floats+static inline float hsum_float_8(const __m256 x) {+ __m128 res = _mm256_extractf128_ps(x, 1);+ res = _mm_add_ps(res, _mm256_castps256_ps128(x));+ res = _mm_add_ps(res, _mm_movehl_ps(res, res));+ res = _mm_add_ss(res, _mm_movehdup_ps(res));+ return _mm_cvtss_f32(res);+}++// horizontally add 8 int32_t+static inline int hsum_i32_8(const __m256i a) {+ const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));+ const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);+ const __m128i sum64 = _mm_add_epi32(hi64, sum128);+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));+}++// horizontally add 4 int32_t+static inline int hsum_i32_4(const __m128i a) {+ const __m128i hi64 = _mm_unpackhi_epi64(a, a);+ const __m128i sum64 = _mm_add_epi32(hi64, a);+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));+}++#if defined(__AVX2__) || defined(__AVX512F__)+static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {+ const __m256i ax = _mm256_sign_epi8(x, x);+ const __m256i sy = _mm256_sign_epi8(y, x);+ return _mm256_maddubs_epi16(ax, sy);+}++// spread 32 bits to 32 bytes { 0x00, 0xFF }+static inline __m256i bytes_from_bits_32(const uint8_t * x) {+ uint32_t x32;+ memcpy(&x32, x, sizeof(uint32_t));+ const __m256i shuf_mask = _mm256_set_epi64x(+ 0x0303030303030303, 0x0202020202020202,+ 0x0101010101010101, 0x0000000000000000);+ __m256i bytes = _mm256_shuffle_epi8(_mm256_set1_epi32(x32), shuf_mask);+ const __m256i bit_mask = _mm256_set1_epi64x(0x7fbfdfeff7fbfdfe);+ bytes = _mm256_or_si256(bytes, bit_mask);+ return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));+}++// Unpack 32 4-bit fields into 32 bytes+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)+{+ const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);+ const __m256i lowMask = _mm256_set1_epi8( 0xF );+ return _mm256_and_si256(lowMask, bytes);+}++// add int16_t pairwise and return as float vector+static inline __m256 sum_i16_pairs_float(const __m256i x) {+ const __m256i ones = _mm256_set1_epi16(1);+ const __m256i summed_pairs = _mm256_madd_epi16(ones, x);+ return _mm256_cvtepi32_ps(summed_pairs);+}++static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {+#if defined(__AVX512VNNI__) && defined(__AVX512VL__)+ const __m256i zero = _mm256_setzero_si256();+ const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);+ return _mm256_cvtepi32_ps(summed_pairs);+#elif defined(__AVXVNNI__)+ const __m256i zero = _mm256_setzero_si256();+ const __m256i summed_pairs = _mm256_dpbusd_avx_epi32(zero, ax, sy);+ return _mm256_cvtepi32_ps(summed_pairs);+#else+ // Perform multiplication and create 16-bit values+ const __m256i dot = _mm256_maddubs_epi16(ax, sy);+ return sum_i16_pairs_float(dot);+#endif+}++// multiply int8_t, add results pairwise twice and return as float vector+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {+#if __AVXVNNIINT8__+ const __m256i zero = _mm256_setzero_si256();+ const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);+ return _mm256_cvtepi32_ps(summed_pairs);+#else+ // Get absolute values of x vectors+ const __m256i ax = _mm256_sign_epi8(x, x);+ // Sign the values of the y vectors+ const __m256i sy = _mm256_sign_epi8(y, x);+ return mul_sum_us8_pairs_float(ax, sy);+#endif+}++static inline __m128i packNibbles( __m256i bytes )+{+ // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh+#if __AVX512F__+ const __m256i bytes_srli_4 = _mm256_srli_epi16(bytes, 4); // 0000_0000_abcd_0000+ bytes = _mm256_or_si256(bytes, bytes_srli_4); // 0000_abcd_abcd_efgh+ return _mm256_cvtepi16_epi8(bytes); // abcd_efgh+#else+ const __m256i lowByte = _mm256_set1_epi16( 0xFF );+ __m256i high = _mm256_andnot_si256( lowByte, bytes );+ __m256i low = _mm256_and_si256( lowByte, bytes );+ high = _mm256_srli_epi16( high, 4 );+ bytes = _mm256_or_si256( low, high );++ // Compress uint16_t lanes into bytes+ __m128i r0 = _mm256_castsi256_si128( bytes );+ __m128i r1 = _mm256_extracti128_si256( bytes, 1 );+ return _mm_packus_epi16( r0, r1 );+#endif+}+#elif defined(__AVX__)+static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )+{+ // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh+ const __m128i lowByte = _mm_set1_epi16( 0xFF );+ __m128i high = _mm_andnot_si128( lowByte, bytes1 );+ __m128i low = _mm_and_si128( lowByte, bytes1 );+ high = _mm_srli_epi16( high, 4 );+ bytes1 = _mm_or_si128( low, high );+ high = _mm_andnot_si128( lowByte, bytes2 );+ low = _mm_and_si128( lowByte, bytes2 );+ high = _mm_srli_epi16( high, 4 );+ bytes2 = _mm_or_si128( low, high );++ return _mm_packus_epi16( bytes1, bytes2);+}++static inline __m128i mul_add_epi8_sse(const __m128i x, const __m128i y) {+ const __m128i ax = _mm_sign_epi8(x, x);+ const __m128i sy = _mm_sign_epi8(y, x);+ return _mm_maddubs_epi16(ax, sy);+}++// spread 32 bits to 32 bytes { 0x00, 0xFF }+static inline __m256i bytes_from_bits_32(const uint8_t * x) {+ uint32_t x32;+ memcpy(&x32, x, sizeof(uint32_t));+ const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);+ const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);+ __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);+ __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);+ const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);+ bytesl = _mm_or_si128(bytesl, bit_mask);+ bytesh = _mm_or_si128(bytesh, bit_mask);+ bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));+ bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));+ return MM256_SET_M128I(bytesh, bytesl);+}++// Unpack 32 4-bit fields into 32 bytes+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)+{+ // Load 16 bytes from memory+ __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);+ __m128i tmph = _mm_srli_epi16(tmpl, 4);+ const __m128i lowMask = _mm_set1_epi8(0xF);+ tmpl = _mm_and_si128(lowMask, tmpl);+ tmph = _mm_and_si128(lowMask, tmph);+ return MM256_SET_M128I(tmph, tmpl);+}++// add int16_t pairwise and return as float vector+static inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {+ const __m128i ones = _mm_set1_epi16(1);+ const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);+ const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);+ const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);+ return _mm256_cvtepi32_ps(summed_pairs);+}++static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {+ const __m128i axl = _mm256_castsi256_si128(ax);+ const __m128i axh = _mm256_extractf128_si256(ax, 1);+ const __m128i syl = _mm256_castsi256_si128(sy);+ const __m128i syh = _mm256_extractf128_si256(sy, 1);+ // Perform multiplication and create 16-bit values+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);+ const __m128i doth = _mm_maddubs_epi16(axh, syh);+ return sum_i16_pairs_float(doth, dotl);+}++// multiply int8_t, add results pairwise twice and return as float vector+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {+ const __m128i xl = _mm256_castsi256_si128(x);+ const __m128i xh = _mm256_extractf128_si256(x, 1);+ const __m128i yl = _mm256_castsi256_si128(y);+ const __m128i yh = _mm256_extractf128_si256(y, 1);+ // Get absolute values of x vectors+ const __m128i axl = _mm_sign_epi8(xl, xl);+ const __m128i axh = _mm_sign_epi8(xh, xh);+ // Sign the values of the y vectors+ const __m128i syl = _mm_sign_epi8(yl, xl);+ const __m128i syh = _mm_sign_epi8(yh, xh);+ // Perform multiplication and create 16-bit values+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);+ const __m128i doth = _mm_maddubs_epi16(axh, syh);+ return sum_i16_pairs_float(doth, dotl);+}++// larger version of mul_sum_i8_pairs_float where x and y are each represented by four 128-bit vectors+static inline __m256 mul_sum_i8_quad_float(const __m128i x_1_0, const __m128i x_1_1, const __m128i x_2_0, const __m128i x_2_1,+ const __m128i y_1_0, const __m128i y_1_1, const __m128i y_2_0, const __m128i y_2_1) {+ const __m128i mone = _mm_set1_epi16(1);++ const __m128i p16_1_0 = mul_add_epi8_sse(x_1_0, y_1_0);+ const __m128i p16_1_1 = mul_add_epi8_sse(x_1_1, y_1_1);+ const __m128i p16_2_0 = mul_add_epi8_sse(x_2_0, y_2_0);+ const __m128i p16_2_1 = mul_add_epi8_sse(x_2_1, y_2_1);+ const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, mone);+ const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, mone);+ const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, mone);+ const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, mone);+ const __m128i p_1 = _mm_add_epi32(p_1_0, p_1_1);+ const __m128i p_2 = _mm_add_epi32(p_2_0, p_2_1);+ return _mm256_cvtepi32_ps(MM256_SET_M128I(p_2, p_1));+}++// quad fp16 delta calculation+static inline __m256 quad_fp16_delta_float(const float x0, const float y0, const float x1, const float y1) {+ // GGML_CPU_FP16_TO_FP32 is faster than Intel F16C+ return _mm256_set_m128(_mm_set1_ps(GGML_CPU_FP16_TO_FP32(x1) * GGML_CPU_FP16_TO_FP32(y1)),+ _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x0) * GGML_CPU_FP16_TO_FP32(y0)));+}++static inline __m256 quad_mx_delta_float(const uint8_t x0, const float y0, const uint8_t x1, const float y1) {+ return _mm256_set_m128(_mm_set1_ps(GGML_CPU_E8M0_TO_FP32_HALF(x1) * GGML_CPU_FP16_TO_FP32(y1)),+ _mm_set1_ps(GGML_CPU_E8M0_TO_FP32_HALF(x0) * GGML_CPU_FP16_TO_FP32(y0)));+}+#endif+#elif defined(__SSSE3__)+static inline __m128i bytes_from_bits_16(const uint8_t * x) {+ uint16_t x16;+ memcpy(&x16, x, sizeof(uint16_t));++ const __m128i shuf_mask = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);+ __m128i bytes = _mm_shuffle_epi8(_mm_set1_epi16((short) x16), shuf_mask);+ const __m128i bit_mask = _mm_set_epi64x(0x7fbfdfeff7fbfdfe, 0x7fbfdfeff7fbfdfe);+ bytes = _mm_or_si128(bytes, bit_mask);++ return _mm_cmpeq_epi8(bytes, _mm_set1_epi64x(-1));+}++// horizontally add 4x4 floats+static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) {+ __m128 res_0 =_mm_hadd_ps(a, b);+ __m128 res_1 =_mm_hadd_ps(c, d);+ __m128 res =_mm_hadd_ps(res_0, res_1);+ res =_mm_hadd_ps(res, res);+ res =_mm_hadd_ps(res, res);++ return _mm_cvtss_f32(res);+}+#endif // __AVX__ || __AVX2__ || __AVX512F__+#endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)++void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ assert(QK8_0 == 32);+ assert(k % QK8_0 == 0);+ const int nb = k / QK8_0;++ block_q8_0 * GGML_RESTRICT y = vy;++#if defined(__AVX2__) || defined(__AVX__)+ for (int i = 0; i < nb; i++) {+ // Load elements into 4 AVX vectors+ __m256 v0 = _mm256_loadu_ps( x );+ __m256 v1 = _mm256_loadu_ps( x + 8 );+ __m256 v2 = _mm256_loadu_ps( x + 16 );+ __m256 v3 = _mm256_loadu_ps( x + 24 );+ x += 32;++ // Compute max(abs(e)) for the block+ const __m256 signBit = _mm256_set1_ps( -0.0f );+ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );++ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );+ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );+ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );+ const float maxScalar = _mm_cvtss_f32( max4 );++ // Quantize these floats+ const float d = maxScalar / 127.f;+ y[i].d = GGML_CPU_FP32_TO_FP16(d);+ const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;+ const __m256 mul = _mm256_set1_ps( id );++ // Apply the multiplier+ v0 = _mm256_mul_ps( v0, mul );+ v1 = _mm256_mul_ps( v1, mul );+ v2 = _mm256_mul_ps( v2, mul );+ v3 = _mm256_mul_ps( v3, mul );++ // Round to nearest integer+ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );+ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );+ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );+ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );++ // Convert floats to integers+ __m256i i0 = _mm256_cvtps_epi32( v0 );+ __m256i i1 = _mm256_cvtps_epi32( v1 );+ __m256i i2 = _mm256_cvtps_epi32( v2 );+ __m256i i3 = _mm256_cvtps_epi32( v3 );++#if defined(__AVX2__)+ // Convert int32 to int16+ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15+ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31+ // Convert int16 to int8+ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31++ // We got our precious signed bytes, but the order is now wrong+ // These AVX2 pack instructions process 16-byte pieces independently+ // The following instruction is fixing the order+ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );+ i0 = _mm256_permutevar8x32_epi32( i0, perm );++ _mm256_storeu_si256((__m256i *)y[i].qs, i0);+#else+ // Since we don't have in AVX some necessary functions,+ // we split the registers in half and call AVX2 analogs from SSE+ __m128i ni0 = _mm256_castsi256_si128( i0 );+ __m128i ni1 = _mm256_extractf128_si256( i0, 1);+ __m128i ni2 = _mm256_castsi256_si128( i1 );+ __m128i ni3 = _mm256_extractf128_si256( i1, 1);+ __m128i ni4 = _mm256_castsi256_si128( i2 );+ __m128i ni5 = _mm256_extractf128_si256( i2, 1);+ __m128i ni6 = _mm256_castsi256_si128( i3 );+ __m128i ni7 = _mm256_extractf128_si256( i3, 1);++ // Convert int32 to int16+ ni0 = _mm_packs_epi32( ni0, ni1 );+ ni2 = _mm_packs_epi32( ni2, ni3 );+ ni4 = _mm_packs_epi32( ni4, ni5 );+ ni6 = _mm_packs_epi32( ni6, ni7 );+ // Convert int16 to int8+ ni0 = _mm_packs_epi16( ni0, ni2 );+ ni4 = _mm_packs_epi16( ni4, ni6 );++ _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);+ _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);+#endif+ }+#else+ GGML_UNUSED(nb);+ // scalar+ quantize_row_q8_0_ref(x, y, k);+#endif+}++void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ assert(k % QK8_1 == 0);+ const int nb = k / QK8_1;++ block_q8_1 * GGML_RESTRICT y = vy;+#if defined(__AVX2__) || defined(__AVX__)+ for (int i = 0; i < nb; i++) {+ // Load elements into 4 AVX vectors+ __m256 v0 = _mm256_loadu_ps( x );+ __m256 v1 = _mm256_loadu_ps( x + 8 );+ __m256 v2 = _mm256_loadu_ps( x + 16 );+ __m256 v3 = _mm256_loadu_ps( x + 24 );+ x += 32;++ // Compute max(abs(e)) for the block+ const __m256 signBit = _mm256_set1_ps( -0.0f );+ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );++ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );+ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );+ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );+ const float max_scalar = _mm_cvtss_f32( max4 );++ // Quantize these floats+ const float d = max_scalar / 127.f;+ y[i].d = GGML_CPU_FP32_TO_FP16(d);+ const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;+ const __m256 mul = _mm256_set1_ps( id );++ // Apply the multiplier+ v0 = _mm256_mul_ps( v0, mul );+ v1 = _mm256_mul_ps( v1, mul );+ v2 = _mm256_mul_ps( v2, mul );+ v3 = _mm256_mul_ps( v3, mul );++ // Round to nearest integer+ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );+ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );+ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );+ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );++ // Convert floats to integers+ __m256i i0 = _mm256_cvtps_epi32( v0 );+ __m256i i1 = _mm256_cvtps_epi32( v1 );+ __m256i i2 = _mm256_cvtps_epi32( v2 );+ __m256i i3 = _mm256_cvtps_epi32( v3 );++#if defined(__AVX2__)+ // Compute the sum of the quants and set y[i].s+ y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))));++ // Convert int32 to int16+ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15+ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31+ // Convert int16 to int8+ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31++ // We got our precious signed bytes, but the order is now wrong+ // These AVX2 pack instructions process 16-byte pieces independently+ // The following instruction is fixing the order+ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );+ i0 = _mm256_permutevar8x32_epi32( i0, perm );++ _mm256_storeu_si256((__m256i *)y[i].qs, i0);+#else+ // Since we don't have in AVX some necessary functions,+ // we split the registers in half and call AVX2 analogs from SSE+ __m128i ni0 = _mm256_castsi256_si128( i0 );+ __m128i ni1 = _mm256_extractf128_si256( i0, 1);+ __m128i ni2 = _mm256_castsi256_si128( i1 );+ __m128i ni3 = _mm256_extractf128_si256( i1, 1);+ __m128i ni4 = _mm256_castsi256_si128( i2 );+ __m128i ni5 = _mm256_extractf128_si256( i2, 1);+ __m128i ni6 = _mm256_castsi256_si128( i3 );+ __m128i ni7 = _mm256_extractf128_si256( i3, 1);++ // Compute the sum of the quants and set y[i].s+ const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3));+ const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7));+ y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_4(_mm_add_epi32(s0, s1)));++ // Convert int32 to int16+ ni0 = _mm_packs_epi32( ni0, ni1 );+ ni2 = _mm_packs_epi32( ni2, ni3 );+ ni4 = _mm_packs_epi32( ni4, ni5 );+ ni6 = _mm_packs_epi32( ni6, ni7 );+ // Convert int16 to int8+ ni0 = _mm_packs_epi16( ni0, ni2 );+ ni4 = _mm_packs_epi16( ni4, ni6 );++ _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);+ _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);+#endif+ }+#else+ GGML_UNUSED(nb);+ // scalar+ quantize_row_q8_1_ref(x, y, k);+#endif+}++// placeholder implementation for Apple targets+void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q8_K_ref(x, y, k);+}++//===================================== Dot products =================================++//+// Helper functions+//++#if __AVX__ || __AVX2__ || __AVX512F__++// shuffles to pick the required scales in dot products+static inline __m256i get_scale_shuffle_q3k(int i) {+ static const uint8_t k_shuffle[128] = {+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,+ };+ return _mm256_loadu_si256((const __m256i*)k_shuffle + i);+}+static inline __m256i get_scale_shuffle_k4(int i) {+ static const uint8_t k_shuffle[256] = {+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,+ 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,+ 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,+ 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,+ 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15+ };+ return _mm256_loadu_si256((const __m256i*)k_shuffle + i);+}+static inline __m128i get_scale_shuffle(int i) {+ static const uint8_t k_shuffle[128] = {+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,+ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,+ 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,+ 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,+ 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,+ 12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,+ 14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15+ };+ return _mm_loadu_si128((const __m128i*)k_shuffle + i);+}+#endif++void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK1_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q1_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++#if defined(__AVX2__)+ const __m256i ones_8 = _mm256_set1_epi8(1);+ const __m256i ones_16 = _mm256_set1_epi16(1);+ const __m256i byte_shuf = _mm256_setr_epi8(+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);+ const __m256i bit_masks = _mm256_setr_epi8(+ 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128,+ 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128);+ const __m256i zero = _mm256_setzero_si256();+ __m256 acc = _mm256_setzero_ps();++ for (int ib = 0; ib < nb; ++ib) {+ const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);+ const uint32_t * GGML_RESTRICT qs32 = (const uint32_t *) x[ib].qs;+ const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4];++ __m256 acc_block;+ {+ const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[0].qs);+ const __m256i sm = _mm256_cmpeq_epi8(+ _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[0]), byte_shuf), bit_masks), zero);+ const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm);+ const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16);+ acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), _mm256_cvtepi32_ps(s32));+ }+ for (int K = 1; K < 4; ++K) {+ const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[K].qs);+ const __m256i sm = _mm256_cmpeq_epi8(+ _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[K]), byte_shuf), bit_masks), zero);+ const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm);+ const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16);+ acc_block = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[K].d)), _mm256_cvtepi32_ps(s32), acc_block);+ }+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc);+ }++ *s = hsum_float_8(acc);+#elif defined(__AVX__)+ const __m128i ones_8 = _mm_set1_epi8(1);+ const __m128i ones_16 = _mm_set1_epi16(1);+ const __m128i zero = _mm_setzero_si128();+ __m256 acc = _mm256_setzero_ps();++ for (int ib = 0; ib < nb; ++ib) {+ const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);+ const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4];+ __m256 acc_block;+ {+ const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[0]);+ const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask);+ const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1);+ const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[0]);+ const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[16]);+ const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero);+ const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero);+ const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0);+ const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1);+ const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0);+ const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1);+ const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16);+ const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16);+ const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0));+ acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), q);+ }+ for(int K = 1; K < 4; ++K) {+ const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[(K) * 4]);+ const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask);+ const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1);+ const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[0]);+ const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[16]);+ const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero);+ const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero);+ const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0);+ const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1);+ const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0);+ const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1);+ const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16);+ const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16);+ const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0));+ acc_block = _mm256_add_ps(acc_block, _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(K)].d)), q));+ }+#undef Q1_AVX_BLOCK++ acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_set1_ps(d0), acc_block));+ }++ *s = hsum_float_8(acc);+#elif defined(__SSSE3__)+ const __m128i ones_8 = _mm_set1_epi8(1);+ const __m128i ones_16 = _mm_set1_epi16(1);+ const __m128i zero = _mm_setzero_si128();+ __m128 acc_0 = _mm_setzero_ps();+ __m128 acc_1 = _mm_setzero_ps();+ __m128 acc_2 = _mm_setzero_ps();+ __m128 acc_3 = _mm_setzero_ps();++ for (int ib = 0; ib < nb; ++ib) {+ const __m128 d0 = _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d));+ const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4];++#define Q1_SSSE3_BLOCK(QS_OFF, Y_IDX, ACC) \+ { \+ const __m128i bit_mask_0 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 0]); \+ const __m128i bit_mask_1 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 2]); \+ const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[0]); \+ const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[16]); \+ const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); \+ const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); \+ const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); \+ const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); \+ const __m128i sum_0 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_0), ones_16); \+ const __m128i sum_1 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_1), ones_16); \+ const __m128 q = _mm_cvtepi32_ps(_mm_add_epi32(sum_0, sum_1)); \+ (ACC) = _mm_add_ps((ACC), _mm_mul_ps(_mm_mul_ps(d0, _mm_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(Y_IDX)].d))), q)); \+ }+ Q1_SSSE3_BLOCK(0, 0, acc_0)+ Q1_SSSE3_BLOCK(4, 1, acc_1)+ Q1_SSSE3_BLOCK(8, 2, acc_2)+ Q1_SSSE3_BLOCK(12, 3, acc_3)+#undef Q1_SSSE3_BLOCK+ }++ *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);+#else+ UNUSED(nb);+ UNUSED(x);+ UNUSED(y);+ ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++#if defined(__AVX2__)+// four consecutive Q2_0 weights per byte: byte b becomes b & 3, (b >> 2) & 3, (b >> 4) & 3, (b >> 6) & 3 (OR, never add: the overlapping copies only touch masked-off bits)+static inline __m256i q2_0_unpack_32_avx2(const uint8_t * qs) {+ __m256i v = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *) qs));+ v = _mm256_or_si256(v, _mm256_slli_epi32(v, 12));+ v = _mm256_or_si256(v, _mm256_slli_epi32(v, 6));+ return _mm256_and_si256(v, _mm256_set1_epi32(0x03030303));+}++// 32 weights against one Q8_0 block: sum((v - 1) * y) = sum(v * y) - sum(y), exact for every int8 y+static inline __m256 q2_0_dot_32_avx2(const uint8_t * qs, const int8_t * qy) {+ const __m256i v = q2_0_unpack_32_avx2(qs);+ const __m256i y = _mm256_loadu_si256((const __m256i *) qy);+ const __m256i p = _mm256_sub_epi16(_mm256_maddubs_epi16(v, y), _mm256_maddubs_epi16(_mm256_set1_epi8(1), y));+ return _mm256_cvtepi32_ps(_mm256_madd_epi16(p, _mm256_set1_epi16(1)));+}++static inline __m256 q2_0_acc_block_avx2(const block_q2_0 * x, const block_q8_0 * y, __m256 acc) {+ const float dx = GGML_CPU_FP16_TO_FP32(x->d);+ const __m256 d0 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[0].d));+ const __m256 d1 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[1].d));+ const __m256 p0 = q2_0_dot_32_avx2(x->qs + 0, y[0].qs);+ const __m256 p1 = q2_0_dot_32_avx2(x->qs + 8, y[1].qs);+#if defined(__FMA__)+ acc = _mm256_fmadd_ps(d0, p0, acc);+ return _mm256_fmadd_ps(d1, p1, acc);+#else+ acc = _mm256_add_ps(acc, _mm256_mul_ps(d0, p0));+ return _mm256_add_ps(acc, _mm256_mul_ps(d1, p1));+#endif+}+#endif++void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+#if defined(__AVX2__)+ assert(n % QK2_0 == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;+ const int nb = n / QK2_0;++ // two Q2_0 blocks per step, each into its own accumulator (two FMA chains)+ __m256 acc0 = _mm256_setzero_ps();+ __m256 acc1 = _mm256_setzero_ps();+ int i = 0;+ for (; i + 1 < nb; i += 2) {+ acc0 = q2_0_acc_block_avx2(x + i + 0, y + 2*i + 0, acc0);+ acc1 = q2_0_acc_block_avx2(x + i + 1, y + 2*i + 2, acc1);+ }+ if (i < nb) {+ acc0 = q2_0_acc_block_avx2(x + i, y + 2*i, acc0);+ }++ *s = hsum_float_8(_mm256_add_ps(acc0, acc1));+#else+ ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ int ib = 0;+ float sumf = 0;++#if defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (; ib < nb; ++ib) {+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );++ __m256i qx = bytes_from_nibbles_32(x[ib].qs);++ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.+ const __m256i off = _mm256_set1_epi8( 8 );+ qx = _mm256_sub_epi8( qx, off );++ __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);++ const __m256 q = mul_sum_i8_pairs_float(qx, qy);++ /* Multiply q with scale and accumulate */+ acc = _mm256_fmadd_ps( d, q, acc );+ }++ sumf = hsum_float_8(acc);+#elif defined(__AVX__)+ __m256 accum = _mm256_setzero_ps();+ for (; ib + 1 < nb; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);++ const __m128i q4b_1_0 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), q4bits_1), _mm_set1_epi8(8));+ const __m128i q4b_1_1 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(q4bits_1, 4)), _mm_set1_epi8(8));+ const __m128i q4b_2_0 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), q4bits_2), _mm_set1_epi8(8));+ const __m128i q4b_2_1 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(q4bits_2, 4)), _mm_set1_epi8(8));++ const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);+ const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);+ const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);+ const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);+ const __m128i p_1 = _mm_add_epi16(p16_1_0, p16_1_1);+ const __m128i p_2 = _mm_add_epi16(p16_2_0, p16_2_1);+ const __m256 p = sum_i16_pairs_float(p_2, p_1);++ const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);+ accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);+ }++ sumf = hsum_float_8(accum);+#elif defined(__SSSE3__)+ // set constants+ const __m128i lowMask = _mm_set1_epi8(0xF);+ const __m128i off = _mm_set1_epi8(8);++ // Initialize accumulator with zeros+ __m128 acc_0 = _mm_setzero_ps();+ __m128 acc_1 = _mm_setzero_ps();+ __m128 acc_2 = _mm_setzero_ps();+ __m128 acc_3 = _mm_setzero_ps();++ for (; ib + 1 < nb; ib += 2) {+ _mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 0 and 1+ const __m128 d_0_1 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );++ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[ib].qs);++ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);+ bx_0 = _mm_sub_epi8(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[ib].qs + 16));+ bx_1 = _mm_sub_epi8(bx_1, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);++ _mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 2 and 3+ const __m128 d_2_3 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) );++ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);++ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);+ bx_2 = _mm_sub_epi8(bx_2, off);+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);++ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[ib + 1].qs + 16));+ bx_3 = _mm_sub_epi8(bx_3, off);+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);++ // Convert int32_t to float+ __m128 p0 = _mm_cvtepi32_ps(i32_0);+ __m128 p1 = _mm_cvtepi32_ps(i32_1);+ __m128 p2 = _mm_cvtepi32_ps(i32_2);+ __m128 p3 = _mm_cvtepi32_ps(i32_3);++ // Apply the scale+ __m128 p0_d = _mm_mul_ps( d_0_1, p0 );+ __m128 p1_d = _mm_mul_ps( d_0_1, p1 );+ __m128 p2_d = _mm_mul_ps( d_2_3, p2 );+ __m128 p3_d = _mm_mul_ps( d_2_3, p3 );++ // Acummulate+ acc_0 = _mm_add_ps(p0_d, acc_0);+ acc_1 = _mm_add_ps(p1_d, acc_1);+ acc_2 = _mm_add_ps(p2_d, acc_2);+ acc_3 = _mm_add_ps(p3_d, acc_3);+ }++ sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);++#endif+ for (; ib < nb; ++ib) {+ int sumi0 = 0;+ int sumi1 = 0;++ for (int j = 0; j < qk/2; ++j) {+ const int v0 = (x[ib].qs[j] & 0x0F) - 8;+ const int v1 = (x[ib].qs[j] >> 4) - 8;++ sumi0 += (v0 * y[ib].qs[j]);+ sumi1 += (v1 * y[ib].qs[j + qk/2]);+ }++ int sumi = sumi0 + sumi1;+ sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);+ }++ *s = sumf;+}++void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_1;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_1 * GGML_RESTRICT x = vx;+ const block_q8_1 * GGML_RESTRICT y = vy;++ int ib = 0;++#if defined(__AVX2__) || defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ float summs = 0;++ // Main loop+ for (; ib < nb; ++ib) {+ const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);+ const float d1 = GGML_CPU_FP16_TO_FP32(y[ib].d);++ summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);++ const __m256 d0v = _mm256_set1_ps( d0 );+ const __m256 d1v = _mm256_set1_ps( d1 );++ // Compute combined scales+ const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );++ // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes+ const __m256i qx = bytes_from_nibbles_32(x[ib].qs);+ const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[ib].qs );++ const __m256 xy = mul_sum_us8_pairs_float(qx, qy);++ // Accumulate d0*d1*x*y+#if defined(__AVX2__)+ acc = _mm256_fmadd_ps( d0d1, xy, acc );+#else+ acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );+#endif+ }++ *s = hsum_float_8(acc) + summs;+#else+ UNUSED(nb);+ UNUSED(x);+ UNUSED(y);+ UNUSED(ib);+ ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK_MXFP4 == 0);+ static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same");++ const block_mxfp4 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ const int nb = n / QK_MXFP4;++ int ib = 0;+ float sumf = 0;++#if defined __AVX2__++ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4);+ const __m128i m4b = _mm_set1_epi8(0x0f);+ const __m256i mone = _mm256_set1_epi16(1);++ __m256 accum1 = _mm256_setzero_ps();+ __m256 accum2 = _mm256_setzero_ps();++ for (; ib + 1 < nb; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs);+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs);+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs);+ const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));+ const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);+ const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);+ const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);+ const __m256 scale0 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib + 0].e));+ const __m256 scale1 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib + 1].e));+ accum1 = _mm256_fmadd_ps(scale0, _mm256_cvtepi32_ps(p_1), accum1);+ accum2 = _mm256_fmadd_ps(scale1, _mm256_cvtepi32_ps(p_2), accum2);+ }++ sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));++#elif defined __AVX__+ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4);+ const __m128i m4b = _mm_set1_epi8(0x0f);++ __m256 accum = _mm256_setzero_ps();+ for (; ib + 1 < nb; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);++ const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));+ const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));+ const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));+ const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));++ const __m256 p = mul_sum_i8_quad_float(q4b_1_0, q4b_1_1, q4b_2_0, q4b_2_1, q8b_1_0, q8b_1_1, q8b_2_0, q8b_2_1);+ const __m256 deltas = quad_mx_delta_float(x[ib].e, y[ib].d, x[ib + 1].e, y[ib + 1].d);+ accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);+ }++ sumf = hsum_float_8(accum);++#endif+ for (; ib < nb; ++ib) {+ const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib].e);+ int sumi1 = 0;+ int sumi2 = 0;+ for (int j = 0; j < QK_MXFP4/2; ++j) {+ sumi1 += y[ib].qs[j + 0] * kvalues_fp4[x[ib].qs[j] & 0xf];+ sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_fp4[x[ib].qs[j] >> 4];+ }+ sumf += d * (sumi1 + sumi2);+ }+ *s = sumf;+}++void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK_NVFP4 == 0);++ const block_nvfp4 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ const int nb = n / QK_NVFP4;+ int ib = 0;+ float sumf = 0;++#if defined(__AVX2__)++ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4);+ const __m128i m4b = _mm_set1_epi8(0x0f);+ const __m256i mone = _mm256_set1_epi16(1);++ __m256 accum = _mm256_setzero_ps();+ for(; ib < nb; ib++){++ const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0));+ const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16));++ const __m256i q8_01 = _mm256_loadu_si256((const __m256i *)y[2*ib + 0].qs);+ const __m256i q8_23 = _mm256_loadu_si256((const __m256i *)y[2*ib + 1].qs);++ const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b));+ const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b));+ const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b));+ const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b));++ //reordering+ const __m256i q4_01 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_01_lo,q4_01_hi), _mm_unpacklo_epi64(q4_01_lo,q4_01_hi));+ const __m256i q4_23 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_23_lo,q4_23_hi),_mm_unpacklo_epi64(q4_23_lo,q4_23_hi));++ const __m256i p01 = mul_add_epi8(q4_01,q8_01);+ const __m256i p_1 = _mm256_madd_epi16(p01, mone);++ const __m256i p23 = mul_add_epi8(q4_23,q8_23);+ const __m256i p_2 = _mm256_madd_epi16(p23, mone);++ const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d);+ const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d);++ const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0;+ const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0;+ const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1;+ const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1;++ const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0));+ const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2));++ accum = _mm256_fmadd_ps(scales01, _mm256_cvtepi32_ps(p_1), accum);+ accum = _mm256_fmadd_ps(scales23, _mm256_cvtepi32_ps(p_2), accum);+ }+ sumf = hsum_float_8(accum);++#elif defined(__AVX__)++ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4);+ const __m128i m4b = _mm_set1_epi8(0x0f);++ __m256 accum = _mm256_setzero_ps();+ for(; ib < nb; ib++){++ const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0));+ const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16));++ const __m128i q8_0 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 0));+ const __m128i q8_1 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 16));+ const __m128i q8_2 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 0));+ const __m128i q8_3 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 16));++ const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b));+ const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b));+ const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b));+ const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b));++ const __m128i q4_0 = _mm_unpacklo_epi64(q4_01_lo, q4_01_hi);+ const __m128i q4_1 = _mm_unpackhi_epi64(q4_01_lo, q4_01_hi);+ const __m128i q4_2 = _mm_unpacklo_epi64(q4_23_lo, q4_23_hi);+ const __m128i q4_3 = _mm_unpackhi_epi64(q4_23_lo, q4_23_hi);++ const __m128i p0_i32 = mul_sum_i8_pairs(q4_0, q8_0);+ const __m128i p1_i32 = mul_sum_i8_pairs(q4_1, q8_1);+ const __m128i p2_i32 = mul_sum_i8_pairs(q4_2, q8_2);+ const __m128i p3_i32 = mul_sum_i8_pairs(q4_3, q8_3);++ const __m128 p0 = _mm_cvtepi32_ps(p0_i32);+ const __m128 p1 = _mm_cvtepi32_ps(p1_i32);+ const __m128 p2 = _mm_cvtepi32_ps(p2_i32);+ const __m128 p3 = _mm_cvtepi32_ps(p3_i32);++ const __m256 p01 = _mm256_set_m128(p1, p0);+ const __m256 p23 = _mm256_set_m128(p3, p2);++ const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d);+ const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d);++ const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0;+ const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0;+ const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1;+ const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1;++ const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0));+ const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2));++ accum = _mm256_add_ps(accum, _mm256_mul_ps(p01, scales01));+ accum = _mm256_add_ps(accum, _mm256_mul_ps(p23, scales23));+ }+ sumf = hsum_float_8(accum);++#endif++ for (;ib < nb; ++ib) {+ for (int s_idx = 0; s_idx < 4; ++s_idx) {+ const float d = GGML_CPU_UE4M3_TO_FP32(x[ib].d[s_idx]);+ const int q8_block = s_idx / 2;+ const int q8_off = (s_idx % 2) * QK_NVFP4_SUB;+ const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8_block].d);++ int sumi_lo = 0, sumi_hi = 0;+ for (int j = 0; j < QK_NVFP4_SUB/2; ++j) {+ const uint8_t qv = x[ib].qs[s_idx*(QK_NVFP4_SUB/2) + j];+ sumi_lo += y[2*ib + q8_block].qs[q8_off + j + 0] * kvalues_fp4[qv & 0xf];+ sumi_hi += y[2*ib + q8_block].qs[q8_off + j + QK_NVFP4_SUB/2] * kvalues_fp4[qv >> 4];+ }++ sumf += dy * d * (sumi_lo + sumi_hi);+ }+ }+ *s = sumf;+}++void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ int ib = 0;++ assert(n % qk == 0);+ assert(qk == QK5_0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q5_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++#if defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (; ib < nb; ++ib) {+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));++ __m256i qx = bytes_from_nibbles_32(x[ib].qs);+ __m256i bxhi = bytes_from_bits_32(x[ib].qh);+ bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));+ qx = _mm256_or_si256(qx, bxhi);++ __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);++ const __m256 q = mul_sum_i8_pairs_float(qx, qy);++ /* Multiply q with scale and accumulate */+ acc = _mm256_fmadd_ps(d, q, acc);+ }++ *s = hsum_float_8(acc);+#elif defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();+ __m128i mask = _mm_set1_epi8((char)0xF0);++ // Main loop+ for (; ib < nb; ++ib) {+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));++ __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);+ const __m256i bxhi = bytes_from_bits_32(x[ib].qh);+ __m128i bxhil = _mm256_castsi256_si128(bxhi);+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);+ bxhil = _mm_andnot_si128(bxhil, mask);+ bxhih = _mm_andnot_si128(bxhih, mask);+ __m128i bxl = _mm256_castsi256_si128(bx_0);+ __m128i bxh = _mm256_extractf128_si256(bx_0, 1);+ bxl = _mm_or_si128(bxl, bxhil);+ bxh = _mm_or_si128(bxh, bxhih);+ bx_0 = MM256_SET_M128I(bxh, bxl);++ const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);++ const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0);++ /* Multiply q with scale and accumulate */+ acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);+ }++ *s = hsum_float_8(acc);+#else+ UNUSED(nb);+ UNUSED(ib);+ UNUSED(x);+ UNUSED(y);+ ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_1;+ const int nb = n / qk;++ int ib = 0;++ assert(n % qk == 0);+ assert(qk == QK5_1);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q5_1 * GGML_RESTRICT x = vx;+ const block_q8_1 * GGML_RESTRICT y = vy;++#if defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ float summs = 0.0f;++ // Main loop+ for (; ib < nb; ++ib) {+ const __m256 dx = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d));++ summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);++ __m256i qx = bytes_from_nibbles_32(x[ib].qs);+ __m256i bxhi = bytes_from_bits_32(x[ib].qh);+ bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));+ qx = _mm256_or_si256(qx, bxhi);++ const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib].d));+ const __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);++ const __m256 q = mul_sum_us8_pairs_float(qx, qy);++ acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);+ }++ *s = hsum_float_8(acc) + summs;+#elif defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();+ __m128i mask = _mm_set1_epi8(0x10);++ float summs = 0.0f;++ // Main loop+ for (; ib < nb; ++ib) {+ const __m256 dx = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d));++ summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);++ __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);+ const __m256i bxhi = bytes_from_bits_32(x[ib].qh);+ __m128i bxhil = _mm256_castsi256_si128(bxhi);+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);+ bxhil = _mm_and_si128(bxhil, mask);+ bxhih = _mm_and_si128(bxhih, mask);+ __m128i bxl = _mm256_castsi256_si128(bx_0);+ __m128i bxh = _mm256_extractf128_si256(bx_0, 1);+ bxl = _mm_or_si128(bxl, bxhil);+ bxh = _mm_or_si128(bxh, bxhih);+ bx_0 = MM256_SET_M128I(bxh, bxl);++ const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib].d));+ const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);++ const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0);++ acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);+ }++ *s = hsum_float_8(acc) + summs;+#else+ UNUSED(nb);+ UNUSED(ib);+ UNUSED(x);+ UNUSED(y);+ ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q8_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ int ib = 0;+ float sumf = 0;++#if defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (; ib < nb; ++ib) {+ // Compute combined scale for the block+ const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));+ __m256i qx = _mm256_loadu_si256((const __m256i *)x[ib].qs);+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);++ const __m256 q = mul_sum_i8_pairs_float(qx, qy);++ // Multiply q with scale and accumulate+ acc = _mm256_fmadd_ps( d, q, acc );+ }++ sumf = hsum_float_8(acc);+#elif defined(__AVX__)+ __m256 accum = _mm256_setzero_ps();++ for (; ib + 1 < nb; ib += 2) {+ const __m128i qx_1_0 = _mm_loadu_si128((const __m128i *)x[ib].qs);+ const __m128i qx_1_1 = _mm_loadu_si128((const __m128i *)x[ib].qs + 1);+ const __m128i qx_2_0 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);+ const __m128i qx_2_1 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs + 1);+ const __m128i qy_1_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);+ const __m128i qy_1_1 = _mm_loadu_si128((const __m128i *)y[ib].qs + 1);+ const __m128i qy_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);+ const __m128i qy_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);++ const __m256 p = mul_sum_i8_quad_float(qx_1_0, qx_1_1, qx_2_0, qx_2_1, qy_1_0, qy_1_1, qy_2_0, qy_2_1);+ const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);+ accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);+ }++ sumf = hsum_float_8(accum);+#endif+ for (; ib < nb; ++ib) {+ int sumi = 0;++ for (int j = 0; j < qk; j++) {+ sumi += x[ib].qs[j]*y[ib].qs[j];+ }++ sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));+ }++ *s = sumf;+}++void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_tq1_0 * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined(__AVX2__)+ __m256 sumf = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {+ // 16-bit sums+ __m256i sumi0 = _mm256_setzero_si256();+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();++ // first 32 bytes of 5 elements+ {+ __m256i qx0 = _mm256_loadu_si256((const __m256i *) (x[i].qs));+ // 8-bit multiplies with shifts, masks and adds+ __m256i qx1 = _mm256_add_epi8(qx0, _mm256_add_epi8(qx0, qx0)); // 1 * 3+ __m256i qx2 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx0, 3), _mm256_set1_epi8(-8)), qx0); // 1 * 9+ __m256i qx3 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx1, 3), _mm256_set1_epi8(-8)), qx1); // 3 * 9+ __m256i qx4 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx2, 3), _mm256_set1_epi8(-8)), qx2); // 9 * 9++ // TODO: can _mm256_mulhi_epu16 be faster even if 16-bits?++ // Cancel the +1 from avg so that it behaves like a halving add+ qx0 = _mm256_subs_epu8(qx0, _mm256_set1_epi8(1));+ qx1 = _mm256_subs_epu8(qx1, _mm256_set1_epi8(1));+ qx2 = _mm256_subs_epu8(qx2, _mm256_set1_epi8(1));+ qx3 = _mm256_subs_epu8(qx3, _mm256_set1_epi8(1));+ qx4 = _mm256_subs_epu8(qx4, _mm256_set1_epi8(1));+ // Multiply by 3 and get the top 2 bits+ qx0 = _mm256_avg_epu8(qx0, _mm256_avg_epu8(qx0, _mm256_setzero_si256()));+ qx1 = _mm256_avg_epu8(qx1, _mm256_avg_epu8(qx1, _mm256_setzero_si256()));+ qx2 = _mm256_avg_epu8(qx2, _mm256_avg_epu8(qx2, _mm256_setzero_si256()));+ qx3 = _mm256_avg_epu8(qx3, _mm256_avg_epu8(qx3, _mm256_setzero_si256()));+ qx4 = _mm256_avg_epu8(qx4, _mm256_avg_epu8(qx4, _mm256_setzero_si256()));+ qx0 = _mm256_and_si256(_mm256_srli_epi16(qx0, 6), _mm256_set1_epi8(3));+ qx1 = _mm256_and_si256(_mm256_srli_epi16(qx1, 6), _mm256_set1_epi8(3));+ qx2 = _mm256_and_si256(_mm256_srli_epi16(qx2, 6), _mm256_set1_epi8(3));+ qx3 = _mm256_and_si256(_mm256_srli_epi16(qx3, 6), _mm256_set1_epi8(3));+ qx4 = _mm256_and_si256(_mm256_srli_epi16(qx4, 6), _mm256_set1_epi8(3));++ const __m256i qy0 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 0));+ const __m256i qy1 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 32));+ const __m256i qy2 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 64));+ const __m256i qy3 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 96));+ const __m256i qy4 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 128));++ qx0 = _mm256_maddubs_epi16(qx0, qy0);+ qx1 = _mm256_maddubs_epi16(qx1, qy1);+ qx2 = _mm256_maddubs_epi16(qx2, qy2);+ qx3 = _mm256_maddubs_epi16(qx3, qy3);+ qx4 = _mm256_maddubs_epi16(qx4, qy4);++ sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(qx0, qx1));+ sumi1 = _mm256_add_epi16(sumi1, _mm256_add_epi16(qx2, qx3));+ sumi2 = _mm256_add_epi16(sumi2, qx4);+ }++ // last 16 bytes of 5-element, along with the 4 bytes of 4 elements+ {+ __m128i qx0 = _mm_loadu_si128((const __m128i *) (x[i].qs + 32));+ uint32_t qh;+ memcpy(&qh, x[i].qh, sizeof(qh)); // potentially unaligned+ __m256i qx5_l = _mm256_cvtepu8_epi16(_mm_set1_epi32(qh));+ __m128i qx1 = _mm_add_epi8(qx0, _mm_add_epi8(qx0, qx0)); // 1 * 3+ __m128i qx2 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx0, 3), _mm_set1_epi8(-8)), qx0); // 1 * 9+ __m128i qx3 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx1, 3), _mm_set1_epi8(-8)), qx1); // 3 * 9+ __m128i qx4 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx2, 3), _mm_set1_epi8(-8)), qx2); // 9 * 9+ __m256i qx01 = MM256_SET_M128I(qx1, qx0);+ __m256i qx23 = MM256_SET_M128I(qx3, qx2);++ // avx2 does not have 8-bit multiplies, so 16-bit it is.+ qx5_l = _mm256_mullo_epi16(qx5_l, _mm256_set_epi16(27, 27, 27, 27, 9, 9, 9, 9, 3, 3, 3, 3, 1, 1, 1, 1));+ qx5_l = _mm256_and_si256(qx5_l, _mm256_set1_epi16(0xFF));+ __m128i qx5 = _mm_packus_epi16(_mm256_castsi256_si128(qx5_l), _mm256_extracti128_si256(qx5_l, 1));++ __m256i qx45 = MM256_SET_M128I(qx5, qx4);++ // Cancel the +1 from avg so that it behaves like a halving add+ qx01 = _mm256_subs_epu8(qx01, _mm256_set1_epi8(1));+ qx23 = _mm256_subs_epu8(qx23, _mm256_set1_epi8(1));+ qx45 = _mm256_subs_epu8(qx45, _mm256_set1_epi8(1));+ // Multiply by 3 and get the top 2 bits+ qx01 = _mm256_avg_epu8(qx01, _mm256_avg_epu8(qx01, _mm256_setzero_si256()));+ qx23 = _mm256_avg_epu8(qx23, _mm256_avg_epu8(qx23, _mm256_setzero_si256()));+ qx45 = _mm256_avg_epu8(qx45, _mm256_avg_epu8(qx45, _mm256_setzero_si256()));+ qx01 = _mm256_and_si256(_mm256_srli_epi16(qx01, 6), _mm256_set1_epi8(3));+ qx23 = _mm256_and_si256(_mm256_srli_epi16(qx23, 6), _mm256_set1_epi8(3));+ qx45 = _mm256_and_si256(_mm256_srli_epi16(qx45, 6), _mm256_set1_epi8(3));++ const __m256i qy01 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 160));+ const __m256i qy23 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 192));+ const __m256i qy45 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 224));++ qx01 = _mm256_maddubs_epi16(qx01, qy01);+ qx23 = _mm256_maddubs_epi16(qx23, qy23);+ qx45 = _mm256_maddubs_epi16(qx45, qy45);++ sumi0 = _mm256_add_epi16(sumi0, qx01);+ sumi1 = _mm256_add_epi16(sumi1, qx23);+ sumi2 = _mm256_add_epi16(sumi2, qx45);+ }++ const __m256i ysum = _mm256_loadu_si256((const __m256i *) y[i].bsums);+ const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d));++ sumi0 = _mm256_sub_epi16(sumi0, ysum);+ sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(sumi1, sumi2));+ sumi0 = _mm256_madd_epi16(sumi0, _mm256_set1_epi16(1));++ sumf = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(sumi0), d), sumf);+ }++ *s = hsum_float_8(sumf);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_tq2_0 * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined(__AVX2__)+ __m256 sumf = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {+ // 16-bit sums, because 256*127 still fits+ __m256i sumi0 = _mm256_setzero_si256();+ __m256i sumi1 = _mm256_setzero_si256();++ for (size_t j = 0; j < sizeof(x->qs); j += 32) {+ __m256i qx0 = _mm256_loadu_si256((const __m256i *) (x[i].qs + j));+ __m256i qx1 = _mm256_srli_epi16(qx0, 2);+ __m256i qx2 = _mm256_srli_epi16(qx0, 4);+ __m256i qx3 = _mm256_srli_epi16(qx0, 6);++ // 0, 1, 2 (should not be 3)+ qx0 = _mm256_and_si256(qx0, _mm256_set1_epi8(3));+ qx1 = _mm256_and_si256(qx1, _mm256_set1_epi8(3));+ qx2 = _mm256_and_si256(qx2, _mm256_set1_epi8(3));+ qx3 = _mm256_and_si256(qx3, _mm256_set1_epi8(3));++ const __m256i qy0 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 0));+ const __m256i qy1 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 32));+ const __m256i qy2 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 64));+ const __m256i qy3 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 96));++ qx0 = _mm256_maddubs_epi16(qx0, qy0);+ qx1 = _mm256_maddubs_epi16(qx1, qy1);+ qx2 = _mm256_maddubs_epi16(qx2, qy2);+ qx3 = _mm256_maddubs_epi16(qx3, qy3);++ sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(qx0, qx1));+ sumi1 = _mm256_add_epi16(sumi1, _mm256_add_epi16(qx2, qx3));+ }++ const __m256i ysum = _mm256_loadu_si256((const __m256i *) y[i].bsums);+ const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d));++ sumi0 = _mm256_add_epi16(sumi0, sumi1);+ sumi0 = _mm256_sub_epi16(sumi0, ysum);+ sumi0 = _mm256_madd_epi16(sumi0, _mm256_set1_epi16(1));++ sumf = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(sumi0), d), sumf);+ }++ *s = hsum_float_8(sumf);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined __AVX2__++ const __m256i m3 = _mm256_set1_epi8(3);+ const __m128i m4 = _mm_set1_epi8(0xF);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);++ const uint8_t * GGML_RESTRICT q2 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);+ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);+ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);+ const __m256i mins = _mm256_cvtepi8_epi16(mins8);+ const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));++ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);++ const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};++ __m256i sumi = _mm256_setzero_si256();++ for (int j = 0; j < QK_K/128; ++j) {++ const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ const __m256i q2_0 = _mm256_and_si256(q2bits, m3);+ const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);+ const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);+ const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);++ __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);+ __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);+ __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);+ __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);++ p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);+ p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);+ p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);+ p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);++ p0 = _mm256_add_epi32(p0, p1);+ p2 = _mm256_add_epi32(p2, p3);++ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));+ }++ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);++ }++ *s = hsum_float_8(acc);++#elif defined __AVX__++ const __m128i m3 = _mm_set1_epi8(0x3);+ const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i m2 = _mm_set1_epi8(0x2);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);++ const uint8_t * GGML_RESTRICT q2 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ // load mins and scales from block_q2_K.scales[QK_K/16]+ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);+ const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);+ const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);+ const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);+ const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));++ // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2+ const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));+ const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));++ // sumf += -dmin * summs in 32bits*8+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);++ const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));+ const __m128i scales[2] = { scales_0, scales_1 };++ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ for (int j = 0; j < QK_K/128; ++j) {++ // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;++ // load 2bits*16*8 from block_q2_K.qs[QK_K/4]+ __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;+ const __m128i q2_0 = _mm_and_si128(q2bits, m3);+ const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);+ const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);+ const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);+ q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;+ const __m128i q2_1 = _mm_and_si128(q2bits, m3);+ const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);+ const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);+ const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);++ // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8+ __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);+ __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);+ __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);+ __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);+ __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);+ __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);+ __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);+ __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);++ // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8+ __m128i shuffle = _mm_set1_epi16(0x0100);+ p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);+ shuffle = _mm_add_epi16(shuffle, m2);+ p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);+ shuffle = _mm_add_epi16(shuffle, m2);+ p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);+ shuffle = _mm_add_epi16(shuffle, m2);+ p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);+ shuffle = _mm_add_epi16(shuffle, m2);+ p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);+ shuffle = _mm_add_epi16(shuffle, m2);+ p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);+ shuffle = _mm_add_epi16(shuffle, m2);+ p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);+ shuffle = _mm_add_epi16(shuffle, m2);+ p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);++ p0 = _mm_add_epi32(p0, p1);+ p2 = _mm_add_epi32(p2, p3);+ p4 = _mm_add_epi32(p4, p5);+ p6 = _mm_add_epi32(p6, p7);++ // isum in 32bits*4*2+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));+ }++ // sumf += dall * isum - dmin * summs in 32bits+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);+ }++ *s = hsum_float_8(acc);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const uint32_t kmask1 = 0x03030303;+ const uint32_t kmask2 = 0x0f0f0f0f;++ const block_q3_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined __AVX2__++ const __m256i m3 = _mm256_set1_epi8(3);+ const __m256i mone = _mm256_set1_epi8(1);+ const __m128i m32 = _mm_set1_epi8(32);++ __m256 acc = _mm256_setzero_ps();++ uint32_t aux[3];++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);++ const uint8_t * GGML_RESTRICT q3 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ // Set up scales+ memcpy(aux, x[i].scales, 12);+ __m128i scales128 = _mm_set_epi32(+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));+ scales128 = _mm_sub_epi8(scales128, m32);+ const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};++ // high bit+ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);++ // integer accumulator+ __m256i sumi = _mm256_setzero_si256();++ int bit = 0;+ int is = 0;++ for (int j = 0; j < QK_K/128; ++j) {+ // load low 2 bits+ const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;++ // prepare low and high bits+ const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);+ const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);+ const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);+ const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);+ const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ // load Q8 quants+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);+ __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);+ __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);+ __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);++ __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);+ __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);+ __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);++ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);++ // multiply with scales+ p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);+ p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);+ p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);+ p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);++ // accumulate+ p16_0 = _mm256_add_epi32(p16_0, p16_1);+ p16_2 = _mm256_add_epi32(p16_2, p16_3);+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));++ }++ // multiply with block scale and accumulate+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);++ }++ *s = hsum_float_8(acc);++#elif defined __AVX__++ const __m128i m3 = _mm_set1_epi8(3);+ const __m128i mone = _mm_set1_epi8(1);+ const __m128i m32 = _mm_set1_epi8(32);+ const __m128i m2 = _mm_set1_epi8(2);++ __m256 acc = _mm256_setzero_ps();++ const uint32_t *aux;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);++ const uint8_t * GGML_RESTRICT q3 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ // Set up scales+ aux = (const uint32_t *)x[i].scales;+ __m128i scales128 = _mm_set_epi32(+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));+ scales128 = _mm_sub_epi8(scales128, m32);+ const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));+ const __m128i scales[2] = { scales_0, scales_1 };++ // high bit *128*2 from block_q3_K.hmask[QK_K/8]+ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);++ // integer accumulator+ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ for (int j = 0; j < QK_K/128; ++j) {+ // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]+ const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;+ const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;++ // prepare low and high bits+ const int bit = j << 2;++ const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);+ const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);+ const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);+ const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);++ const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);+ const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);+ const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);+ const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);++ const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);+ const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);+ const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);+ const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);++ const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);+ const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);+ const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);+ const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);++ // load Q8 quants from block_q8_K.qs[QK_K]+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);+ __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);+ __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);+ __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);+ __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);+ __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);+ __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);+ __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);++ __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);+ __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);+ __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);+ __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);+ __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);+ __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);+ __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);+ __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);++ p16_0 = _mm_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);++ // multiply with scales+ __m128i shuffle = _mm_set1_epi16(0x0100);+ p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);++ // accumulate+ p16_0 = _mm_add_epi32(p16_0, p16_1);+ p16_2 = _mm_add_epi32(p16_2, p16_3);+ p16_4 = _mm_add_epi32(p16_4, p16_5);+ p16_6 = _mm_add_epi32(p16_6, p16_7);+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));++ }++ // multiply with block scale and accumulate+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);++ }++ *s = hsum_float_8(acc);++#else+ UNUSED(kmask1);+ UNUSED(kmask2);+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;++ uint32_t utmp[4];++#if defined __AVX2__++ const __m256i m4 = _mm256_set1_epi8(0xF);++ __m256 acc = _mm256_setzero_ps();+ __m128 acc_m = _mm_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);++ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ const uint8_t * GGML_RESTRICT q4 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));++ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);+ acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);++ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);+ const __m256i scales = MM256_SET_M128I(sc128, sc128);++ __m256i sumi = _mm256_setzero_si256();++ for (int j = 0; j < QK_K/64; ++j) {++ const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));+ const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));++ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4l = _mm256_and_si256(q4bits, m4);+ const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);++ const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);+ p16l = _mm256_madd_epi16(scale_l, p16l);++ const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);+ p16h = _mm256_madd_epi16(scale_h, p16h);+ const __m256i sumj = _mm256_add_epi32(p16l, p16h);++ sumi = _mm256_add_epi32(sumi, sumj);+ }++ __m256 vd = _mm256_set1_ps(d);+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);++ }++ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));++ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);++#elif defined __AVX__++ const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i m2 = _mm_set1_epi8(0x2);++ __m256 acc = _mm256_setzero_ps();+ __m128 acc_m = _mm_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);++ const uint8_t * GGML_RESTRICT q4 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);+ const __m128i scales = _mm_cvtepu8_epi16(utmps);+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));++ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);+ const __m128i prod = _mm_madd_epi16(mins, q8s);+ acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);++ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ __m128i shuffle = _mm_set1_epi16(0x0100);+ for (int j = 0; j < QK_K/64; ++j) {++ const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);+ const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);++ __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4l_0 = _mm_and_si128(q4bits, m4);+ const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);+ q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4l_1 = _mm_and_si128(q4bits, m4);+ const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);++ const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);+ p16l = _mm_madd_epi16(scale_l, p16l);+ sumi_0 = _mm_add_epi32(sumi_0, p16l);+ const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ p16l = _mm_maddubs_epi16(q4l_1, q8l_1);+ p16l = _mm_madd_epi16(scale_l, p16l);+ sumi_1 = _mm_add_epi32(sumi_1, p16l);++ const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);+ p16h = _mm_madd_epi16(scale_h, p16h);+ sumi_0 = _mm_add_epi32(sumi_0, p16h);+ const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ p16h = _mm_maddubs_epi16(q4h_1, q8h_1);+ p16h = _mm_madd_epi16(scale_h, p16h);+ sumi_1 = _mm_add_epi32(sumi_1, p16h);++ }++ __m256 vd = _mm256_set1_ps(d);+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);++ }++ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));++ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ UNUSED(kmask1);+ UNUSED(kmask2);+ UNUSED(kmask3);+ UNUSED(utmp);+ ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q5_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;++ uint32_t utmp[4];++#if defined __AVX2__++ const __m256i m4 = _mm256_set1_epi8(0xF);+ const __m128i mzero = _mm_setzero_si128();+ const __m256i mone = _mm256_set1_epi8(1);++ __m256 acc = _mm256_setzero_ps();++ float summs = 0.f;++ for (int i = 0; i < nb; ++i) {+ const uint8_t * GGML_RESTRICT q5 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);++ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));++ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);+ summs += dmin * _mm_extract_epi32(hsum, 0);++ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);+ const __m256i scales = MM256_SET_M128I(sc128, sc128);++ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);+ __m256i hmask = mone;++ __m256i sumi = _mm256_setzero_si256();++ int bit = 0;++ for (int j = 0; j < QK_K/64; ++j) {++ const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));+ const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));++ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;++ const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);+ const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);+ const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);+ hmask = _mm256_slli_epi16(hmask, 1);++ const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);+ const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);+ const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);+ hmask = _mm256_slli_epi16(hmask, 1);++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);++ p16_0 = _mm256_madd_epi16(scale_0, p16_0);+ p16_1 = _mm256_madd_epi16(scale_1, p16_1);++ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));++ }++ __m256 vd = _mm256_set1_ps(d);+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);++ }++ *s = hsum_float_8(acc) + summs;++#elif defined __AVX__++ const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i mzero = _mm_setzero_si128();+ const __m128i mone = _mm_set1_epi8(1);+ const __m128i m2 = _mm_set1_epi8(2);++ __m256 acc = _mm256_setzero_ps();++ float summs = 0.f;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);++ const uint8_t * GGML_RESTRICT q5 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);+ const __m128i scales = _mm_cvtepu8_epi16(utmps);+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));++ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);+ const __m128i prod = _mm_madd_epi16(mins, q8s);+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);+ summs += dmin * _mm_extract_epi32(hsum, 0);++ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);+ __m128i hmask = mone;++ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ int bit = 0;++ __m128i shuffle = _mm_set1_epi16(0x0100);+ for (int j = 0; j < QK_K/64; ++j) {++ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);++ const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;+ const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;++ __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);+ __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);+ __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);+ __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);+ __m128i q5_0 = _mm_add_epi8(q5l_0, q5h_0);+ __m128i q5_1 = _mm_add_epi8(q5l_1, q5h_1);+ hmask = _mm_slli_epi16(hmask, 1);++ __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);+ __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);+ p16_0 = _mm_madd_epi16(scale_0, p16_0);+ p16_1 = _mm_madd_epi16(scale_0, p16_1);++ q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);+ q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);+ q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);+ q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);+ q5_0 = _mm_add_epi8(q5l_0, q5h_0);+ q5_1 = _mm_add_epi8(q5l_1, q5h_1);+ hmask = _mm_slli_epi16(hmask, 1);++ q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);+ __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);+ p16_2 = _mm_madd_epi16(scale_1, p16_2);+ p16_3 = _mm_madd_epi16(scale_1, p16_3);++ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));++ }++ __m256 vd = _mm256_set1_ps(d);+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);++ }++ *s = hsum_float_8(acc) + summs;++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ UNUSED(kmask1);+ UNUSED(kmask2);+ UNUSED(kmask3);+ UNUSED(utmp);+ ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q6_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined __AVX2__++ const __m256i m3 = _mm256_set1_epi8(3);+ const __m256i m15 = _mm256_set1_epi8(15);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);++ const uint8_t * GGML_RESTRICT q4 = x[i].ql;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);+ const __m256i scales_16 = _mm256_cvtepi8_epi16(scales);+ const __m256i q8sclsub = _mm256_slli_epi32(_mm256_madd_epi16(q8sums, scales_16), 5);++ __m256i sumi = _mm256_setzero_si256();++ int is = 0;++ for (int j = 0; j < QK_K/128; ++j) {+ const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;++ const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m3), 4);+ const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, _mm256_set1_epi8(12)), 2);+ const __m256i q4h_2 = _mm256_and_si256(q4bitsH, _mm256_set1_epi8(48));+ const __m256i q4h_3 = _mm256_srli_epi16(_mm256_and_si256(q4bitsH, _mm256_set1_epi8(-64)), 2);++ const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m15), q4h_0);+ const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m15), q4h_1);+ const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m15), q4h_2);+ const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m15), q4h_3);++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);+ __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);+ __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);++ const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));+ const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));+ const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));+ const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));+ is += 4;++ p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);+ p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);+ p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);+ p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);++ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));++ }++ sumi = _mm256_sub_epi32(sumi, q8sclsub);+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);+ }++ *s = hsum_float_8(acc);++#elif defined __AVX__++ const __m128i m3 = _mm_set1_epi8(3);+ const __m128i m15 = _mm_set1_epi8(15);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);++ const uint8_t * GGML_RESTRICT q4 = x[i].ql;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ // handle the q6_k -32 offset separately using bsums+ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)y[i].bsums);+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)y[i].bsums + 1);+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);+ const __m128i scales_16_0 = _mm_cvtepi8_epi16(scales);+ const __m128i scales_16_1 = _mm_cvtepi8_epi16(_mm_bsrli_si128(scales, 8));+ const __m128i q8sclsub_0 = _mm_slli_epi32(_mm_madd_epi16(q8sums_0, scales_16_0), 5);+ const __m128i q8sclsub_1 = _mm_slli_epi32(_mm_madd_epi16(q8sums_1, scales_16_1), 5);++ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ int is = 0;++ for (int j = 0; j < QK_K/128; ++j) {++ const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;+ const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;++ const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);+ const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);+ const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, _mm_set1_epi8(12)), 2);+ const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, _mm_set1_epi8(12)), 2);+ const __m128i q4h_4 = _mm_and_si128(q4bitsH_0, _mm_set1_epi8(48));+ const __m128i q4h_5 = _mm_and_si128(q4bitsH_1, _mm_set1_epi8(48));+ const __m128i q4h_6 = _mm_srli_epi16(_mm_and_si128(q4bitsH_0, _mm_set1_epi8(-64)), 2);+ const __m128i q4h_7 = _mm_srli_epi16(_mm_and_si128(q4bitsH_1, _mm_set1_epi8(-64)), 2);++ const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;++ const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m15), q4h_0);+ const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m15), q4h_1);+ const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m15), q4h_2);+ const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m15), q4h_3);+ const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m15), q4h_4);+ const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m15), q4h_5);+ const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m15), q4h_6);+ const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m15), q4h_7);++ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;++ __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);+ __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);+ __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);+ __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);+ __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);+ __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);+ __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);+ __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);++ const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));+ const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));+ const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));+ const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));+ is += 4;++ p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);+ p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_0, 8)), p16_1);+ p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);+ p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_1, 8)), p16_3);+ p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);+ p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_2, 8)), p16_5);+ p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);+ p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_3, 8)), p16_7);++ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));++ }++ sumi_0 = _mm_sub_epi32(sumi_0, q8sclsub_0);+ sumi_1 = _mm_sub_epi32(sumi_1, q8sclsub_1);+ const __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi)), acc);+ }++ *s = hsum_float_8(acc);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++#if defined (__AVX__) || defined (__AVX2__)+static const int8_t keven_signs_q2xs[1024] = {+ 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,+ 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,+ 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,+ 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,+ 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,+ 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,+ 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,+ 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,+ 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,+ 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,+ 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,+ 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,+ 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,+ 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,+ 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,+ 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,+ 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,+ 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,+ 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,+ 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,+ 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,+ 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,+ 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,+ 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,+ 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,+ 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,+ 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,+ 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,+ 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,+ 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,+ 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,+ 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,+};+#endif++void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq2_xxs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined(__AVX2__)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[4];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * GGML_RESTRICT q2 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;+ const __m256i q2_1 = _mm256_set_epi64x(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);+ const __m256i q2_2 = _mm256_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);+ const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],+ signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const uint16_t ls1 = aux32[1] >> 28;+ const uint16_t ls2 = aux32[3] >> 28;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));+ sumi1 = _mm256_add_epi32(sumi1, p1);+ sumi2 = _mm256_add_epi32(sumi2, p2);+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);++#elif defined(__AVX__)+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[4];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * GGML_RESTRICT q2 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ __m128i sumi2_0 = _mm_setzero_si128();+ __m128i sumi2_1 = _mm_setzero_si128();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;+ const __m128i q2_1_0 = _mm_set_epi64x(iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);+ const __m128i q2_1_1 = _mm_set_epi64x(iq2xxs_grid[aux8[3]], iq2xxs_grid[aux8[2]]);+ const __m128i q2_2_0 = _mm_set_epi64x(iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);+ const __m128i q2_2_1 = _mm_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]]);+ const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);+ const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);+ const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127]);+ const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);+ const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);+ const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);+ const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);+ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);+ const uint16_t ls1 = aux32[1] >> 28;+ const uint16_t ls2 = aux32[3] >> 28;+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);+ }++ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq2_xs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined(__AVX2__)++ const __m256i mone = _mm256_set1_epi8(1);+ static const char block_sign_shuffle_mask_1[32] = {+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,+ 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,+ };+ static const char block_sign_shuffle_mask_2[32] = {+ 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,+ };+ static const uint8_t bit_selector_mask_bytes[32] = {+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m256i bit_selector_mask = _mm256_loadu_si256((const __m256i*)bit_selector_mask_bytes);+ const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1);+ const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2);++ static const uint8_t k_bit_helper[32] = {+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ };+ const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper);+ const __m256i m511 = _mm256_set1_epi16(511);+ const __m128i m4 = _mm_set1_epi8(0xf);+ const __m128i m1 = _mm_set1_epi8(1);++ uint64_t aux64;++ // somewhat hacky, but gives a significant boost in performance+ __m256i aux_gindex;+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * GGML_RESTRICT q2 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ memcpy(&aux64, x[i].scales, 8);+ __m128i stmp = _mm_set1_epi64x(aux64);+ stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));+ const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);++ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {++ const __m256i q2_data = _mm256_loadu_si256((const __m256i*)q2); q2 += 16;+ aux_gindex = _mm256_and_si256(q2_data, m511);++ const __m256i partial_sign_bits = _mm256_srli_epi16(q2_data, 9);+ const __m256i partial_sign_bits_upper = _mm256_srli_epi16(q2_data, 13);+ const __m256i partial_sign_bits_for_counting = _mm256_xor_si256(partial_sign_bits, partial_sign_bits_upper);++ const __m256i odd_bits = _mm256_shuffle_epi8(bit_helper, partial_sign_bits_for_counting);+ const __m256i full_sign_bits = _mm256_or_si256(partial_sign_bits, odd_bits);++ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_4 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;++ const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],+ iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);+ const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],+ iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);+ const __m256i q2_3 = _mm256_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],+ iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);+ const __m256i q2_4 = _mm256_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],+ iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);++ const __m128i full_signs_l = _mm256_castsi256_si128(full_sign_bits);+ const __m128i full_signs_h = _mm256_extractf128_si256(full_sign_bits, 1);+ const __m256i full_signs_1 = MM256_SET_M128I(full_signs_l, full_signs_l);+ const __m256i full_signs_2 = MM256_SET_M128I(full_signs_h, full_signs_h);++ __m256i signs;+ signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_1);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone));++ signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_2);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone));++ signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_1);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_3 = _mm256_sign_epi8(q8_3, _mm256_or_si256(signs, mone));++ signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_2);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_4 = _mm256_sign_epi8(q8_4, _mm256_or_si256(signs, mone));++ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const __m256i dot3 = _mm256_maddubs_epi16(q2_3, q8s_3);+ const __m256i dot4 = _mm256_maddubs_epi16(q2_4, q8s_4);++ const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0)));+ const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1)));+ const __m256i sc3 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2)));+ const __m256i sc4 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3)));++ sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1));+ sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2));+ sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot3, sc3));+ sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot4, sc4));+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);++#elif defined(__AVX__)+ const __m128i mone = _mm_set1_epi8(1);+ static const char block_sign_shuffle_mask_1[32] = {+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,+ 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,+ };+ static const char block_sign_shuffle_mask_2[32] = {+ 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,+ };+ static const uint8_t bit_selector_mask_bytes[32] = {+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m128i bit_selector_mask_0 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes);+ const __m128i bit_selector_mask_1 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes + 1);+ const __m128i block_sign_shuffle_1_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1);+ const __m128i block_sign_shuffle_1_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1 + 1);+ const __m128i block_sign_shuffle_2_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2);+ const __m128i block_sign_shuffle_2_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2 + 1);++ static const uint8_t k_bit_helper[32] = {+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ };+ const __m128i bit_helper_0 = _mm_loadu_si128((const __m128i*)k_bit_helper);+ const __m128i bit_helper_1 = _mm_loadu_si128((const __m128i*)k_bit_helper + 1);+ const __m128i m511 = _mm_set1_epi16(511);+ const __m128i m4 = _mm_set1_epi8(0xf);+ const __m128i m1 = _mm_set1_epi8(1);++ uint64_t aux64;++ // somewhat hacky, but gives a significant boost in performance+ __m256i aux_gindex;+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * GGML_RESTRICT q2 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ memcpy(&aux64, x[i].scales, 8);+ __m128i stmp = _mm_set1_epi64x(aux64);+ stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));+ const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);++ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ __m128i sumi2_0 = _mm_setzero_si128();+ __m128i sumi2_1 = _mm_setzero_si128();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {++ const __m128i q2_data_0 = _mm_loadu_si128((const __m128i*)q2);+ const __m128i q2_data_1 = _mm_loadu_si128((const __m128i*)q2 + 1); q2 += 16;+ aux_gindex = MM256_SET_M128I(_mm_and_si128(q2_data_1, m511), _mm_and_si128(q2_data_0, m511));++ const __m128i partial_sign_bits_0 = _mm_srli_epi16(q2_data_0, 9);+ const __m128i partial_sign_bits_1 = _mm_srli_epi16(q2_data_1, 9);+ const __m128i partial_sign_bits_upper_0 = _mm_srli_epi16(q2_data_0, 13);+ const __m128i partial_sign_bits_upper_1 = _mm_srli_epi16(q2_data_1, 13);+ const __m128i partial_sign_bits_for_counting_0 = _mm_xor_si128(partial_sign_bits_0, partial_sign_bits_upper_0);+ const __m128i partial_sign_bits_for_counting_1 = _mm_xor_si128(partial_sign_bits_1, partial_sign_bits_upper_1);++ const __m128i odd_bits_0 = _mm_shuffle_epi8(bit_helper_0, partial_sign_bits_for_counting_0);+ const __m128i odd_bits_1 = _mm_shuffle_epi8(bit_helper_1, partial_sign_bits_for_counting_1);+ const __m128i full_sign_bits_0 = _mm_or_si128(partial_sign_bits_0, odd_bits_0);+ const __m128i full_sign_bits_1 = _mm_or_si128(partial_sign_bits_1, odd_bits_1);++ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_3_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_3_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_4_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_4_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;++ const __m128i q2_1_0 = _mm_set_epi64x(iq2xs_grid[gindex[1]], iq2xs_grid[gindex[0]]);+ const __m128i q2_1_1 = _mm_set_epi64x(iq2xs_grid[gindex[3]], iq2xs_grid[gindex[2]]);+ const __m128i q2_2_0 = _mm_set_epi64x(iq2xs_grid[gindex[5]], iq2xs_grid[gindex[4]]);+ const __m128i q2_2_1 = _mm_set_epi64x(iq2xs_grid[gindex[7]], iq2xs_grid[gindex[6]]);+ const __m128i q2_3_0 = _mm_set_epi64x(iq2xs_grid[gindex[9]], iq2xs_grid[gindex[8]]);+ const __m128i q2_3_1 = _mm_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]]);+ const __m128i q2_4_0 = _mm_set_epi64x(iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);+ const __m128i q2_4_1 = _mm_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]]);++ // AVX2 full_signs_1 is full_sign_bits_0 here+ // AVX2 full_signs_2 is full_sign_bits_1 here+ __m128i signs_0, signs_1;+ signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_0);+ signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_1);+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);+ const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, _mm_or_si128(signs_0, mone));+ const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, _mm_or_si128(signs_1, mone));++ signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_0);+ signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_1);+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);+ const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, _mm_or_si128(signs_0, mone));+ const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, _mm_or_si128(signs_1, mone));++ signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_0);+ signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_1);+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);+ const __m128i q8s_3_0 = _mm_sign_epi8(q8_3_0, _mm_or_si128(signs_0, mone));+ const __m128i q8s_3_1 = _mm_sign_epi8(q8_3_1, _mm_or_si128(signs_1, mone));++ signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_0);+ signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_1);+ signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);+ signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);+ const __m128i q8s_4_0 = _mm_sign_epi8(q8_4_0, _mm_or_si128(signs_0, mone));+ const __m128i q8s_4_1 = _mm_sign_epi8(q8_4_1, _mm_or_si128(signs_1, mone));++ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);+ const __m128i dot3_0 = _mm_maddubs_epi16(q2_3_0, q8s_3_0);+ const __m128i dot3_1 = _mm_maddubs_epi16(q2_3_1, q8s_3_1);+ const __m128i dot4_0 = _mm_maddubs_epi16(q2_4_0, q8s_4_0);+ const __m128i dot4_1 = _mm_maddubs_epi16(q2_4_1, q8s_4_1);++ __m128i sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0));+ const __m128i sc1_0 = _mm_cvtepi8_epi16(sc_tmp);+ const __m128i sc1_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));+ sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1));+ const __m128i sc2_0 = _mm_cvtepi8_epi16(sc_tmp);+ const __m128i sc2_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));+ sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2));+ const __m128i sc3_0 = _mm_cvtepi8_epi16(sc_tmp);+ const __m128i sc3_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));+ sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3));+ const __m128i sc4_0 = _mm_cvtepi8_epi16(sc_tmp);+ const __m128i sc4_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));++ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot1_0, sc1_0));+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot1_1, sc1_1));+ sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot2_0, sc2_0));+ sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot2_1, sc2_1));+ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot3_0, sc3_0));+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot3_1, sc3_1));+ sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot4_0, sc4_0));+ sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot4_1, sc4_1));+ }++ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq2_s * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined(__AVX2__)++ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m128i m4 = _mm_set1_epi8(0xf);+ const __m128i m1 = _mm_set1_epi8(1);++ const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);+ const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);++ uint64_t aux64;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT qs = x[i].qs;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ memcpy(&aux64, x[i].scales, 8);+ const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);+ const __m256i scales16 = _mm256_cvtepi8_epi16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15++ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q2_1 = _mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],+ iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],+ iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],+ iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);+ const __m256i q2_2 = _mm256_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],+ iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],+ iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],+ iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);+ qs += 8;++ __m256i aux256 = _mm256_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);+ const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);+ const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);++ aux256 = _mm256_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);+ const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);+ const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);++ signs += 4;++ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3++ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+0)));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+1)));+ sumi1 = _mm256_add_epi32(sumi1, p1);+ sumi2 = _mm256_add_epi32(sumi2, p2);+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);++#elif defined(__AVX__)+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m128i m4 = _mm_set1_epi8(0xf);+ const __m128i m1 = _mm_set1_epi8(1);++ const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);+ const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);+ const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);+ const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);++ uint64_t aux64;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT qs = x[i].qs;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);+ const int8_t * GGML_RESTRICT q8 = y[i].qs;++ memcpy(&aux64, x[i].scales, 8);+ const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);+ const __m128i scales16_0 = _mm_cvtepi8_epi16(scales8);+ const __m128i scales16_1 = _mm_cvtepi8_epi16(_mm_srli_si128(scales8, 8));++ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ __m128i sumi2_0 = _mm_setzero_si128();+ __m128i sumi2_1 = _mm_setzero_si128();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q2_1_0 = _mm_set_epi64x(iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],+ iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);+ const __m128i q2_1_1 = _mm_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],+ iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)]);+ const __m128i q2_2_0 = _mm_set_epi64x(iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],+ iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);+ const __m128i q2_2_1 = _mm_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],+ iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)]);+ qs += 8;++ __m128i aux128_0 = _mm_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));+ __m128i aux128_1 = aux128_0;+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);+ const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);+ const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);+ const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);+ const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);++ aux128_0 = _mm_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));+ aux128_1 = aux128_0;+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);+ const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);+ const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);+ const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);+ const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);++ signs += 4;++ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);++ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 0)));+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 1)));+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 0)));+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 1)));+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);+ }++ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq3_xxs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined(__AVX2__)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[2];++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT q3 = x[i].qs;+ const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q2_1 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ q3 += 8;+ const __m256i q2_2 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ q3 += 8;+ memcpy(aux32, gas, 8); gas += 8;+ const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],+ signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);+ const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const uint16_t ls1 = aux32[0] >> 28;+ const uint16_t ls2 = aux32[1] >> 28;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));+ sumi1 = _mm256_add_epi32(sumi1, p1);+ sumi2 = _mm256_add_epi32(sumi2, p2);+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.25f * hsum_float_8(accumf);++#elif defined(__AVX__)+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[2];++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT q3 = x[i].qs;+ const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ __m128i sumi2_0 = _mm_setzero_si128();+ __m128i sumi2_1 = _mm_setzero_si128();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q2_1_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ const __m128i q2_1_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);+ q3 += 8;+ const __m128i q2_2_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ const __m128i q2_2_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);+ q3 += 8;+ memcpy(aux32, gas, 8); gas += 8;+ const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);+ const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127]);+ const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);+ const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);+ const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);+ const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);+ const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);+ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);+ const uint16_t ls1 = aux32[0] >> 28;+ const uint16_t ls2 = aux32[1] >> 28;+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);+ }++ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);++ }++ *s = 0.25f * hsum_float_8(accumf);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq3_s * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined(__AVX2__)++ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);+ const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);++ const __m256i idx_shift = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);+ const __m256i idx_mask = _mm256_set1_epi32(256);++ typedef union {+ __m256i vec[2];+ uint32_t index[16];+ } index_t;++ index_t idx;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT qs = x[i].qs;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i idx_l = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)qs)); qs += 16;+ idx.vec[0] = _mm256_set1_epi32(qh[ib32+0]);+ idx.vec[1] = _mm256_set1_epi32(qh[ib32+1]);+ idx.vec[0] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[0], idx_shift), idx_mask);+ idx.vec[1] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[1], idx_shift), idx_mask);+ idx.vec[0] = _mm256_or_si256(idx.vec[0], _mm256_cvtepi16_epi32(_mm256_castsi256_si128(idx_l)));+ idx.vec[1] = _mm256_or_si256(idx.vec[1], _mm256_cvtepi16_epi32(_mm256_extractf128_si256(idx_l, 1)));++ // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.+ //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);+ //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);+ const __m256i q2_1 = _mm256_set_epi32(+ iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],+ iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]+ );+ const __m256i q2_2 = _mm256_set_epi32(+ iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],+ iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]+ );++ __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16));+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);+ const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);+ const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);++ aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16));+ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);+ const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);+ const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);++ signs += 4;++ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;+ const uint16_t ls2 = x[i].scales[ib32/2] >> 4;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));+ sumi1 = _mm256_add_epi32(sumi1, p1);+ sumi2 = _mm256_add_epi32(sumi2, p2);+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = hsum_float_8(accumf);++#elif defined(__AVX__)+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);+ const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);+ const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);+ const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);++ const __m128i idx_mul_0 = _mm_set_epi32(32, 64, 128, 256);+ const __m128i idx_mul_1 = _mm_set_epi32(2, 4, 8, 16);+ const __m128i idx_mask = _mm_set1_epi32(256);++ typedef union {+ __m128i vec[4];+ uint32_t index[16];+ } index_t;++ index_t idx;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT qs = x[i].qs;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ __m128i sumi2_0 = _mm_setzero_si128();+ __m128i sumi2_1 = _mm_setzero_si128();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i qs_tmp = _mm_loadu_si128((const __m128i *)qs);+ const __m128i idx_l_0 = _mm_cvtepu8_epi16(qs_tmp);+ const __m128i idx_l_1 = _mm_cvtepu8_epi16(_mm_srli_si128(qs_tmp, 8)); qs += 16;+ idx.vec[0] = _mm_set1_epi32(qh[ib32+0]);+ idx.vec[1] = idx.vec[0];+ idx.vec[2] = _mm_set1_epi32(qh[ib32+1]);+ idx.vec[3] = idx.vec[2];++ idx.vec[0] = _mm_and_si128(_mm_mullo_epi32(idx.vec[0], idx_mul_0), idx_mask);+ idx.vec[1] = _mm_and_si128(_mm_mullo_epi32(idx.vec[1], idx_mul_1), idx_mask);+ idx.vec[2] = _mm_and_si128(_mm_mullo_epi32(idx.vec[2], idx_mul_0), idx_mask);+ idx.vec[3] = _mm_and_si128(_mm_mullo_epi32(idx.vec[3], idx_mul_1), idx_mask);++ idx.vec[0] = _mm_or_si128(idx.vec[0], _mm_cvtepi16_epi32(idx_l_0));+ idx.vec[1] = _mm_or_si128(idx.vec[1], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_0, 8)));+ idx.vec[2] = _mm_or_si128(idx.vec[2], _mm_cvtepi16_epi32(idx_l_1));+ idx.vec[3] = _mm_or_si128(idx.vec[3], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_1, 8)));++ const __m128i q2_1_0 = _mm_set_epi32(iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]);+ const __m128i q2_1_1 = _mm_set_epi32(iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]]);+ const __m128i q2_2_0 = _mm_set_epi32(iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[9]], iq3s_grid[idx.index[8]]);+ const __m128i q2_2_1 = _mm_set_epi32(iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]]);++ __m128i aux128_0 = _mm_set1_epi32(signs[0] | (signs[1] << 16));+ __m128i aux128_1 = aux128_0;+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);+ const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);+ const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);+ const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);+ const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);++ aux128_0 = _mm_set1_epi32(signs[2] | (signs[3] << 16));+ aux128_1 = aux128_0;+ aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);+ aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);+ const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);+ const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);+ const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);+ const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);++ signs += 4;++ const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);+ const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);+ const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);+ const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);+ const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;+ const uint16_t ls2 = x[i].scales[ib32/2] >> 4;+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));+ sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);+ sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);+ sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);+ sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);+ }++ accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);++ }++ *s = hsum_float_8(accumf);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq1_s * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined __AVX2__++ __m256 accum = _mm256_setzero_ps();+ float accum1 = 0;+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;++ __m256i sumi = _mm256_setzero_si256();+ int sumi1 = 0;+ for (int ib = 0; ib < QK_K/32; ib += 2) {+#ifdef __BMI2__+ const uint64_t packed_idx1 = _pdep_u64(*(const uint32_t *)qs, 0x00ff00ff00ff00ffULL) | _pdep_u64(qh[ib], 0x700070007000700ULL);+ const uint64_t packed_idx2 = _pdep_u64(*(const uint32_t *)(qs + 4), 0x00ff00ff00ff00ffULL) | _pdep_u64(qh[ib + 1], 0x700070007000700ULL);+ const uint16_t *idx1 = (const uint16_t *)(&packed_idx1);+ const uint16_t *idx2 = (const uint16_t *)(&packed_idx2);+ const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[idx1[3]], iq1s_grid[idx1[2]], iq1s_grid[idx1[1]], iq1s_grid[idx1[0]]);+ const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[idx2[3]], iq1s_grid[idx2[2]], iq1s_grid[idx2[1]], iq1s_grid[idx2[0]]);+#else+ const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)],+ iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);+ const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)],+ iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);+#endif+ qs += 8;+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);+ const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);+ const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;+ const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(ls1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(ls2));++ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p1, p2));+ sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;+ }++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ accum = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), accum);+ accum1 += d * sumi1;++ }++ *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;++#elif defined __AVX__+ __m256 accum = _mm256_setzero_ps();+ float accum1 = 0;+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;++ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ int sumi1 = 0;+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ const __m128i q1b_1_0 = _mm_set_epi64x(iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);+ const __m128i q1b_1_1 = _mm_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)]);+ const __m128i q1b_2_0 = _mm_set_epi64x(iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);+ const __m128i q1b_2_1 = _mm_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)]);+ qs += 8;+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;++ const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);+ const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);+ const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);+ const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);+ const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;+ const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(ls1));+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(ls1));+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(ls2));+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(ls2));++ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));+ sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;+ }++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum);+ accum1 += d * sumi1;++ }++ *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq1_m * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ iq1m_scale_t scale;++#if defined __AVX2__++ const __m256i mask = _mm256_set1_epi16(0x7);+ const __m256i mone = _mm256_set1_epi16(1);+ const __m256i mone8 = _mm256_set1_epi8(1);+ const __m256i mtwo8 = _mm256_set1_epi8(2);+ // VPSHUFB cannot cross 128-bit lanes so odd shifts go to upper half.+ const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);++ __m256 accum1 = _mm256_setzero_ps();+ __m256 accum2 = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint8_t * qh = x[i].qh;+ const uint16_t * sc = (const uint16_t *)x[i].scales;++ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);+ // Extract 3-bit scales (16 values)+ __m256i scales = _mm256_set1_epi64x(*(const uint64_t*)sc);+ scales = _mm256_srlv_epi64(scales, scales_shift);+ scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);++ // Indices to repeat each scale 8 times.+ __m256i scales_idx1 = _mm256_set1_epi16(0x0100);+ __m256i scales_idx2 = _mm256_add_epi8(scales_idx1, _mm256_set1_epi8(8));++ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib = 0; ib < QK_K/32; ib += 2) {+#ifdef __BMI2__+ const uint64_t packed_idx1 = _pdep_u64(*(const uint32_t *)qs, 0x00ff00ff00ff00ffULL)+ | _pdep_u64(*(const uint16_t*)(qh) & 0x7777, 0xf000f000f000f00ULL);+ const uint64_t packed_idx2 = _pdep_u64(*(const uint32_t *)(qs + 4), 0x00ff00ff00ff00ffULL)+ | _pdep_u64(*(const uint16_t*)(qh + 2) & 0x7777, 0xf000f000f000f00ULL);+ const uint16_t *idx1 = (const uint16_t *)(&packed_idx1);+ const uint16_t *idx2 = (const uint16_t *)(&packed_idx2);+ const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[idx1[3]], iq1s_grid[idx1[2]], iq1s_grid[idx1[1]], iq1s_grid[idx1[0]]);+ const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[idx2[3]], iq1s_grid[idx2[2]], iq1s_grid[idx2[1]], iq1s_grid[idx2[0]]);++ // Convert signs to bytes 0x81 (negative) or 0x01 (positive)+ const uint64_t delta_sign = _pdep_u64(*(const uint32_t*)(qh) & 0x88888888, 0xf0f0f0f0f0f0f0f0ULL);+ const __m256i delta1 = _mm256_or_si256(mone8, _mm256_cvtepi8_epi64(_mm_set1_epi32(delta_sign)));+ const __m256i delta2 = _mm256_or_si256(mone8, _mm256_cvtepi8_epi64(_mm_set1_epi32(delta_sign >> 32)));+#else+ const __m256i q1b_1 = _mm256_set_epi64x(+ iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)],+ iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]+ );+ const __m256i q1b_2 = _mm256_set_epi64x(+ iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)],+ iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]+ );++ const __m256i delta1 = _mm256_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);+ const __m256i delta2 = _mm256_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);+#endif+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);+ const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);+ const __m256i dot3 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_1, delta1));+ const __m256i dot4 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_2, delta2));++ __m256i scale1 = _mm256_shuffle_epi8(scales, scales_idx1);+ __m256i scale2 = _mm256_shuffle_epi8(scales, scales_idx2);++ scales_idx1 = _mm256_add_epi8(scales_idx1, mtwo8);+ scales_idx2 = _mm256_add_epi8(scales_idx2, mtwo8);++ const __m256i p1 = _mm256_madd_epi16(dot1, scale1);+ const __m256i p2 = _mm256_madd_epi16(dot2, scale2);+ const __m256i p3 = _mm256_madd_epi16(dot3, scale1);+ const __m256i p4 = _mm256_madd_epi16(dot4, scale2);++ sumi1 = _mm256_add_epi32(sumi1, _mm256_add_epi32(p1, p2));+ sumi2 = _mm256_add_epi32(sumi2, _mm256_add_epi32(p3, p4));++ qs += 8; qh += 4;+ }++ const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16));++ accum1 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi1), accum1);+ accum2 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi2), accum2);+ }++ *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);++#elif defined __AVX__+ const __m128i mask = _mm_set1_epi16(0x7);+ const __m128i mone = _mm_set1_epi16(1);++ __m256 accum1 = _mm256_setzero_ps();+ __m256 accum2 = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint8_t * qh = x[i].qh;+ const uint16_t * sc = (const uint16_t *)x[i].scales;++ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);++ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ __m128i sumi2_0 = _mm_setzero_si128();+ __m128i sumi2_1 = _mm_setzero_si128();+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ const __m128i q1b_1_0 = _mm_set_epi64x(+ iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]);+ const __m128i q1b_1_1 = _mm_set_epi64x(+ iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)]);+ const __m128i q1b_2_0 = _mm_set_epi64x(+ iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]);+ const __m128i q1b_2_1 = _mm_set_epi64x(+ iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)]);+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;++ const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);+ const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);+ const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);+ const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);++ const __m128i delta1_0 = _mm_set_epi64x(qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);+ const __m128i delta1_1 = _mm_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);+ const __m128i delta2_0 = _mm_set_epi64x(qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);+ const __m128i delta2_1 = _mm_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,+ qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);++ const __m128i dot3_0 = mul_add_epi8_sse(delta1_0, q8b_1_0);+ const __m128i dot3_1 = mul_add_epi8_sse(delta1_1, q8b_1_1);+ const __m128i dot4_0 = mul_add_epi8_sse(delta2_0, q8b_2_0);+ const __m128i dot4_1 = mul_add_epi8_sse(delta2_1, q8b_2_1);++ __m128i scale1_0 = _mm_set1_epi16(sc[ib/2] >> 0);+ __m128i scale1_1 = _mm_set1_epi16(sc[ib/2] >> 3);+ __m128i scale2_0 = _mm_set1_epi16(sc[ib/2] >> 6);+ __m128i scale2_1 = _mm_set1_epi16(sc[ib/2] >> 9);++ scale1_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_0, mask), 1), mone);+ scale1_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_1, mask), 1), mone);+ scale2_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_0, mask), 1), mone);+ scale2_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_1, mask), 1), mone);+ const __m128i p1_0 = _mm_madd_epi16(dot1_0, scale1_0);+ const __m128i p1_1 = _mm_madd_epi16(dot1_1, scale1_1);+ const __m128i p2_0 = _mm_madd_epi16(dot2_0, scale2_0);+ const __m128i p2_1 = _mm_madd_epi16(dot2_1, scale2_1);+ const __m128i p3_0 = _mm_madd_epi16(dot3_0, scale1_0);+ const __m128i p3_1 = _mm_madd_epi16(dot3_1, scale1_1);+ const __m128i p4_0 = _mm_madd_epi16(dot4_0, scale2_0);+ const __m128i p4_1 = _mm_madd_epi16(dot4_1, scale2_1);++ sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));+ sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));+ sumi2_0 = _mm_add_epi32(sumi2_0, _mm_add_epi32(p3_0, p4_0));+ sumi2_1 = _mm_add_epi32(sumi2_1, _mm_add_epi32(p3_1, p4_1));++ qs += 8; qh += 4;+ }++ const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16));++ accum1 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum1);+ accum2 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi2_1, sumi2_0))), accum2);+ }++ *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ UNUSED(scale);+ ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}++void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK4_NL == 0);+ static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same");++ const block_iq4_nl * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ const int nb = n / QK4_NL;++ int ib = 0;+ float sumf = 0;++#if defined __AVX2__++ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);+ const __m128i m4b = _mm_set1_epi8(0x0f);+ const __m256i mone = _mm256_set1_epi16(1);++ __m256 accum1 = _mm256_setzero_ps();+ __m256 accum2 = _mm256_setzero_ps();+ for (; ib + 1 < nb; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs);+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs);+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs);+ const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));+ const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);+ const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);+ const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);+ accum1 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)),+ _mm256_cvtepi32_ps(p_1), accum1);+ accum2 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)),+ _mm256_cvtepi32_ps(p_2), accum2);+ }++ sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));++#elif defined __AVX__+ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);+ const __m128i m4b = _mm_set1_epi8(0x0f);++ __m256 accum = _mm256_setzero_ps();+ for (; ib + 1 < nb; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);++ const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));+ const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));+ const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));+ const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));++ const __m256 p = mul_sum_i8_quad_float(q4b_1_0, q4b_1_1, q4b_2_0, q4b_2_1, q8b_1_0, q8b_1_1, q8b_2_0, q8b_2_1);+ const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);+ accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);+ }++ sumf = hsum_float_8(accum);++#endif+ for (; ib < nb; ++ib) {+ const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < QK4_NL/2; ++j) {+ sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];+ sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];+ }+ sumf += d * (sumi1 + sumi2);+ }+ *s = sumf;+}++void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK_K == 0);++ const block_iq4_xs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++#if defined __AVX2__++ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);+ const __m128i m4b = _mm_set1_epi8(0x0f);++ __m256 accum = _mm256_setzero_ps();+ for (int ibl = 0; ibl < nb; ++ibl) {+ const uint8_t * qs = x[ibl].qs;+ const int8_t * q8 = y[ibl].qs;+ uint16_t sh = x[ibl].scales_h;+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)qs); qs += 16;+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)qs); qs += 16;+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));+ const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);+ const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;+ const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;+ sh >>= 4;+ const __m256i p_1 = _mm256_madd_epi16(p16_1, _mm256_set1_epi16(ls1));+ const __m256i p_2 = _mm256_madd_epi16(p16_2, _mm256_set1_epi16(ls2));+ sumi1 = _mm256_add_epi32(p_1, sumi1);+ sumi2 = _mm256_add_epi32(p_2, sumi2);+ }+ accum = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),+ _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accum);+ }++ *s = hsum_float_8(accum);++#elif defined __AVX__+ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);+ const __m128i m4b = _mm_set1_epi8(0x0f);++ __m256 accum = _mm256_setzero_ps();+ for (int ibl = 0; ibl < nb; ++ibl) {+ const uint8_t * qs = x[ibl].qs;+ const int8_t * q8 = y[ibl].qs;+ uint16_t sh = x[ibl].scales_h;+ __m128i sumi1_0 = _mm_setzero_si128();+ __m128i sumi1_1 = _mm_setzero_si128();+ __m128i sumi2_0 = _mm_setzero_si128();+ __m128i sumi2_1 = _mm_setzero_si128();+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)qs); qs += 16;+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)qs); qs += 16;+ const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;+ const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));+ const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));+ const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));+ const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));+ const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);+ const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);+ const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);+ const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);+ const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;+ const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;+ sh >>= 4;+ const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, _mm_set1_epi16(ls1));+ const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, _mm_set1_epi16(ls1));+ const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, _mm_set1_epi16(ls2));+ const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, _mm_set1_epi16(ls2));+ sumi1_0 = _mm_add_epi32(p_1_0, sumi1_0);+ sumi1_1 = _mm_add_epi32(p_1_1, sumi1_1);+ sumi2_0 = _mm_add_epi32(p_2_0, sumi2_0);+ sumi2_1 = _mm_add_epi32(p_2_1, sumi2_1);+ }+ __m128i sumi12_0 = _mm_add_epi32(sumi1_0, sumi2_0);+ __m128i sumi12_1 = _mm_add_epi32(sumi1_1, sumi2_1);+ accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),+ _mm256_cvtepi32_ps(MM256_SET_M128I(sumi12_1, sumi12_0))), accum);+ }++ *s = hsum_float_8(accum);++#else+ UNUSED(x);+ UNUSED(y);+ UNUSED(nb);+ ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/ggml-cpu-impl.h b/python/freetoken/kernel/csrc/ggml/ggml-cpu/ggml-cpu-impl.hnew file mode 100644index 00000000..82345b20--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/ggml-cpu-impl.h@@ -0,0 +1,538 @@+#pragma once++// GGML CPU internal header++#include "ggml.h"+#include "ggml-impl.h"++#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/+//#include <stddef.h>+#include <stdbool.h>+#include <string.h> // memcpy+#include <math.h> // fabsf++#ifdef __cplusplus+extern "C" {+#endif++struct ggml_compute_params {+ // ith = thread index, nth = number of threads+ int ith, nth;++ // work buffer for all threads+ size_t wsize;+ void * wdata;++ struct ggml_threadpool * threadpool;++ // use reference implementation+ bool use_ref;+};+++#if defined(_MSC_VER)++#define m512bh(p) p+#define m512i(p) p++#else++#define m512bh(p) (__m512bh)(p)+#define m512i(p) (__m512i)(p)++#endif++// __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512+#if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))+#ifndef __FMA__+#define __FMA__+#endif+#ifndef __F16C__+#define __F16C__+#endif+#endif++// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available+#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))+#ifndef __SSE3__+#define __SSE3__+#endif+#ifndef __SSSE3__+#define __SSSE3__+#endif+#endif++#if defined(__s390x__) && defined(__VEC__)+#ifndef __VXE__+#define __VXE__+#endif // __VXE__+#ifndef __VXE2__+#define __VXE2__+#endif // __VXE2__+#endif // __s390x__ && __VEC__++#if defined(__ARM_FEATURE_SVE) && defined(__linux__)+#include <sys/prctl.h>+#endif++#if defined(__ARM_NEON)++// ref: https://github.com/ggml-org/llama.cpp/pull/5404+#if defined(_MSC_VER) && !defined(__clang__)+#define ggml_vld1q_u32(w,x,y,z) { ((w) + ((uint64_t)(x) << 32)), ((y) + ((uint64_t)(z) << 32)) }+#else+#define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) }+#endif // _MSC_VER++#if !defined(__aarch64__)++// 32-bit ARM compatibility++// vaddlvq_s16+// vpaddq_s16+// vpaddq_s32+// vaddvq_s32+// vaddvq_f32+// vmaxvq_f32+// vcvtnq_s32_f32+// vzip1_u8+// vzip2_u8++inline static int32_t vaddlvq_s16(int16x8_t v) {+ int32x4_t v0 = vreinterpretq_s32_s64(vpaddlq_s32(vpaddlq_s16(v)));+ return vgetq_lane_s32(v0, 0) + vgetq_lane_s32(v0, 2);+}++inline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) {+ int16x4_t a0 = vpadd_s16(vget_low_s16(a), vget_high_s16(a));+ int16x4_t b0 = vpadd_s16(vget_low_s16(b), vget_high_s16(b));+ return vcombine_s16(a0, b0);+}++inline static int32x4_t vpaddq_s32(int32x4_t a, int32x4_t b) {+ int32x2_t a0 = vpadd_s32(vget_low_s32(a), vget_high_s32(a));+ int32x2_t b0 = vpadd_s32(vget_low_s32(b), vget_high_s32(b));+ return vcombine_s32(a0, b0);+}++inline static int32_t vaddvq_s32(int32x4_t v) {+ return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3);+}++inline static float vaddvq_f32(float32x4_t v) {+ return vgetq_lane_f32(v, 0) + vgetq_lane_f32(v, 1) + vgetq_lane_f32(v, 2) + vgetq_lane_f32(v, 3);+}++inline static float vmaxvq_f32(float32x4_t v) {+ return+ MAX(MAX(vgetq_lane_f32(v, 0), vgetq_lane_f32(v, 1)),+ MAX(vgetq_lane_f32(v, 2), vgetq_lane_f32(v, 3)));+}++inline static int32x4_t vcvtnq_s32_f32(float32x4_t v) {+ int32x4_t res;++ res[0] = roundf(vgetq_lane_f32(v, 0));+ res[1] = roundf(vgetq_lane_f32(v, 1));+ res[2] = roundf(vgetq_lane_f32(v, 2));+ res[3] = roundf(vgetq_lane_f32(v, 3));++ return res;+}++inline static uint8x8_t vzip1_u8(uint8x8_t a, uint8x8_t b) {+ uint8x8_t res;++ res[0] = a[0]; res[1] = b[0];+ res[2] = a[1]; res[3] = b[1];+ res[4] = a[2]; res[5] = b[2];+ res[6] = a[3]; res[7] = b[3];++ return res;+}++inline static uint8x8_t vzip2_u8(uint8x8_t a, uint8x8_t b) {+ uint8x8_t res;++ res[0] = a[4]; res[1] = b[4];+ res[2] = a[5]; res[3] = b[5];+ res[4] = a[6]; res[5] = b[6];+ res[6] = a[7]; res[7] = b[7];++ return res;+}++// vld1q_s16_x2+// vld1q_u8_x2+// vld1q_u8_x4+// vld1q_s8_x2+// vld1q_s8_x4+// TODO: double-check these work correctly++typedef struct ggml_int16x8x2_t {+ int16x8_t val[2];+} ggml_int16x8x2_t;++inline static ggml_int16x8x2_t ggml_vld1q_s16_x2(const int16_t * ptr) {+ ggml_int16x8x2_t res;++ res.val[0] = vld1q_s16(ptr + 0);+ res.val[1] = vld1q_s16(ptr + 8);++ return res;+}++typedef struct ggml_uint8x16x2_t {+ uint8x16_t val[2];+} ggml_uint8x16x2_t;++inline static ggml_uint8x16x2_t ggml_vld1q_u8_x2(const uint8_t * ptr) {+ ggml_uint8x16x2_t res;++ res.val[0] = vld1q_u8(ptr + 0);+ res.val[1] = vld1q_u8(ptr + 16);++ return res;+}++typedef struct ggml_uint8x16x4_t {+ uint8x16_t val[4];+} ggml_uint8x16x4_t;++inline static ggml_uint8x16x4_t ggml_vld1q_u8_x4(const uint8_t * ptr) {+ ggml_uint8x16x4_t res;++ res.val[0] = vld1q_u8(ptr + 0);+ res.val[1] = vld1q_u8(ptr + 16);+ res.val[2] = vld1q_u8(ptr + 32);+ res.val[3] = vld1q_u8(ptr + 48);++ return res;+}++typedef struct ggml_int8x16x2_t {+ int8x16_t val[2];+} ggml_int8x16x2_t;++inline static ggml_int8x16x2_t ggml_vld1q_s8_x2(const int8_t * ptr) {+ ggml_int8x16x2_t res;++ res.val[0] = vld1q_s8(ptr + 0);+ res.val[1] = vld1q_s8(ptr + 16);++ return res;+}++typedef struct ggml_int8x16x4_t {+ int8x16_t val[4];+} ggml_int8x16x4_t;++inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) {+ ggml_int8x16x4_t res;++ res.val[0] = vld1q_s8(ptr + 0);+ res.val[1] = vld1q_s8(ptr + 16);+ res.val[2] = vld1q_s8(ptr + 32);+ res.val[3] = vld1q_s8(ptr + 48);++ return res;+}++// NOTE: not tested+inline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) {+ int8x16_t res;++ res[ 0] = a[b[ 0]];+ res[ 1] = a[b[ 1]];+ res[ 2] = a[b[ 2]];+ res[ 3] = a[b[ 3]];+ res[ 4] = a[b[ 4]];+ res[ 5] = a[b[ 5]];+ res[ 6] = a[b[ 6]];+ res[ 7] = a[b[ 7]];+ res[ 8] = a[b[ 8]];+ res[ 9] = a[b[ 9]];+ res[10] = a[b[10]];+ res[11] = a[b[11]];+ res[12] = a[b[12]];+ res[13] = a[b[13]];+ res[14] = a[b[14]];+ res[15] = a[b[15]];++ return res;+}++// NOTE: not tested+inline static uint8x16_t ggml_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) {+ uint8x16_t res;++ res[ 0] = a[b[ 0]];+ res[ 1] = a[b[ 1]];+ res[ 2] = a[b[ 2]];+ res[ 3] = a[b[ 3]];+ res[ 4] = a[b[ 4]];+ res[ 5] = a[b[ 5]];+ res[ 6] = a[b[ 6]];+ res[ 7] = a[b[ 7]];+ res[ 8] = a[b[ 8]];+ res[ 9] = a[b[ 9]];+ res[10] = a[b[10]];+ res[11] = a[b[11]];+ res[12] = a[b[12]];+ res[13] = a[b[13]];+ res[14] = a[b[14]];+ res[15] = a[b[15]];++ return res;+}++#else++#define ggml_int16x8x2_t int16x8x2_t+#define ggml_uint8x16x2_t uint8x16x2_t+#define ggml_uint8x16x4_t uint8x16x4_t+#define ggml_int8x16x2_t int8x16x2_t+#define ggml_int8x16x4_t int8x16x4_t++#define ggml_vld1q_s16_x2 vld1q_s16_x2+#define ggml_vld1q_u8_x2 vld1q_u8_x2+#define ggml_vld1q_u8_x4 vld1q_u8_x4+#define ggml_vld1q_s8_x2 vld1q_s8_x2+#define ggml_vld1q_s8_x4 vld1q_s8_x4+#define ggml_vqtbl1q_s8 vqtbl1q_s8+#define ggml_vqtbl1q_u8 vqtbl1q_u8++#endif // !defined(__aarch64__)++#if !defined(__ARM_FEATURE_DOTPROD)++// NOTE: this fallback produces the same total sum as native vdotq_s32 but with different per-lane grouping — do not use when individual lane values matter.+inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) {+ const int16x8_t p0 = vmull_s8(vget_low_s8 (a), vget_low_s8 (b));+ const int16x8_t p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b));++ return vaddq_s32(acc, vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)));+}++#else++#define ggml_vdotq_s32(a, b, c) vdotq_s32(a, b, c)++#endif // !defined(__ARM_FEATURE_DOTPROD)++static inline int32x4_t ggml_nvfp4_dot8(const int8x8_t q4_lo, const int8x8_t q8_lo,+ const int8x8_t q4_hi, const int8x8_t q8_hi) {+ const int16x8_t p_lo = vmull_s8(q4_lo, q8_lo);+ const int16x8_t p_hi = vmull_s8(q4_hi, q8_hi);+ const int32x4_t sum_lo = vpaddlq_s16(p_lo);+ const int32x4_t sum_hi = vpaddlq_s16(p_hi);+ return vaddq_s32(sum_lo, sum_hi);+}++#endif // defined(__ARM_NEON)++#ifdef __wasm_simd128__+#include <wasm_simd128.h>+#endif++#ifdef __POWER9_VECTOR__+#include <altivec.h>+#endif++#if defined(_MSC_VER) || defined(__MINGW32__)+#include <intrin.h>+#elif defined(__SSE__) || defined(__SSE3__) || defined(__SSSE3__) || defined(__AVX__) || defined(__F16C__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX512BF16__)+#include <immintrin.h>+#endif++#ifdef __riscv_v_intrinsic+#include <riscv_vector.h>+#endif++#if defined(__loongarch64)+#if defined(__loongarch_asx)+#include <lasxintrin.h>+#endif+#if defined(__loongarch_sx)+#include <lsxintrin.h>+#endif+#endif++#if defined(__VXE__) || defined(__VXE2__)+#include <vecintrin.h>++#define vec_neg(a) (-(a)) // Vector Negate+#define vec_add(a, b) ((a) + (b)) // Vector Add+#define vec_sub(a, b) ((a) - (b)) // Vector Subtract+#define vec_mul(a, b) ((a) * (b)) // Vector Multiply+#define vec_div(a, b) ((a) / (b)) // Vector Divide+#define vec_sl(a, b) ((a) << (b)) // Vector Shift Left+#define vec_sra(a, b) ((a) >> (b)) // Vector Shift Right+#define vec_sr(a, b) ((a) >> (b)) // Vector Shift Right Algebraic+#define vec_slo(a, b) vec_slb(a, (b) << 64) // Vector Shift Left by Octet+#define vec_sro(a, b) vec_srb(a, (b) << 64) // Vector Shift Right by Octet++#ifndef vec_and+#define vec_and(a, b) ((a) & (b)) // Vector AND+#endif++#ifndef vec_or+#define vec_or(a, b) ((a) | (b)) // Vector OR+#endif++#ifndef vec_xor+#define vec_xor(a, b) ((a) ^ (b)) // Vector XOR+#endif++typedef signed char char8x16_t __attribute__((vector_size(16)));+typedef unsigned char uchar8x16_t __attribute__((vector_size(16)));++typedef int8_t int8x16_t __attribute__((vector_size(16)));+typedef int16_t int16x8_t __attribute__((vector_size(16)));+typedef int32_t int32x4_t __attribute__((vector_size(16)));+typedef int64_t int64x2_t __attribute__((vector_size(16)));++typedef uint8_t uint8x16_t __attribute__((vector_size(16)));+typedef uint16_t uint16x8_t __attribute__((vector_size(16)));+typedef uint32_t uint32x4_t __attribute__((vector_size(16)));+typedef uint64_t uint64x2_t __attribute__((vector_size(16)));++typedef float float32x4_t __attribute__((vector_size(16)));+typedef double double64x2_t __attribute__((vector_size(16)));++typedef struct ggml_uint8x16x2_t {+ uint8x16_t val[2];+} ggml_uint8x16x2_t;++inline static ggml_uint8x16x2_t ggml_vec_xl_u8x2(const uint8_t * ptr) {+ ggml_uint8x16x2_t res;++ res.val[0] = vec_xl( 0, ptr);+ res.val[1] = vec_xl(16, ptr);++ return res;+}++typedef struct ggml_uint8x16x4_t {+ uint8x16_t val[4];+} ggml_uint8x16x4_t;++inline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) {+ ggml_uint8x16x4_t res;++ res.val[0] = vec_xl( 0, ptr);+ res.val[1] = vec_xl(16, ptr);+ res.val[2] = vec_xl(32, ptr);+ res.val[3] = vec_xl(48, ptr);++ return res;+}++typedef struct ggml_int8x16x4_t {+ int8x16_t val[4];+} ggml_int8x16x4_t;++inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) {+ ggml_int8x16x4_t res;++ res.val[0] = vec_xl( 0, ptr);+ res.val[1] = vec_xl(16, ptr);+ res.val[2] = vec_xl(32, ptr);+ res.val[3] = vec_xl(48, ptr);++ return res;+}++typedef struct ggml_int16x8x2_t {+ int16x8_t val[2];+} ggml_int16x8x2_t;++inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) {+ ggml_int16x8x2_t res;++ res.val[0] = vec_xl( 0, ptr);+ res.val[1] = vec_xl(16, ptr);++ return res;+}++/*+ ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs+ ! or iq4_nl for example implementation.+*/+inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) {+ int8x16_t res;++ res[ 0] = a[b[ 0]];+ res[ 1] = a[b[ 1]];+ res[ 2] = a[b[ 2]];+ res[ 3] = a[b[ 3]];+ res[ 4] = a[b[ 4]];+ res[ 5] = a[b[ 5]];+ res[ 6] = a[b[ 6]];+ res[ 7] = a[b[ 7]];+ res[ 8] = a[b[ 8]];+ res[ 9] = a[b[ 9]];+ res[10] = a[b[10]];+ res[11] = a[b[11]];+ res[12] = a[b[12]];+ res[13] = a[b[13]];+ res[14] = a[b[14]];+ res[15] = a[b[15]];++ return res;+}++inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) {+ const uchar8x16_t v_maske = { 0, 1, 4, 5, 8, 9, 12, 13,+ 16, 17, 20, 21, 24, 25, 28, 29 };++ const int16x8_t v_abo = vec_pack((int32x4_t)a, (int32x4_t)b);+ const int16x8_t v_abe = vec_perm(a, b, v_maske);+ return v_abo + v_abe;+}++/**+ * @see https://github.com/ggml-org/llama.cpp/pull/14037+ */+inline static float vec_hsum_f32x4(float32x4_t v) {+ float32x4_t v_temp = v + vec_reve(v);+ return v_temp[0] + v_temp[1];+}++inline static int32_t vec_hsum_i32x4(int32x4_t v) {+ int32x4_t v_temp = v + vec_reve(v);+ return v_temp[0] + v_temp[1];+}++inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) {+ const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b);+ return acc + (vec_unpackh(p) + vec_unpackl(p));+}++#endif++#if defined(__loongarch_sx)+/* float type data load instructions */+static __m128 __lsx_vreplfr2vr_s(const float val) {+ v4f32 res = {val, val, val, val};+ return (__m128)res;+}+#endif++#if defined(__loongarch_asx)+static __m256 __lasx_xvreplfr2vr_s(const float val) {+ v8f32 res = {val, val, val, val, val, val, val, val};+ return (__m256)res;+}+#endif++// TODO: move to ggml-threading+void ggml_barrier(struct ggml_threadpool * tp);++void ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value);+int ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/quants.c b/python/freetoken/kernel/csrc/ggml/ggml-cpu/quants.cnew file mode 100644index 00000000..5e36459f--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/quants.c@@ -0,0 +1,1339 @@+#define GGML_COMMON_IMPL_C+#include "ggml-common.h"++#include "ggml-cpu-impl.h"+#include "simd-mappings.h"+#include "ggml-quants.h"+#include "quants.h"++#include "arch-fallback.h"++#include <string.h>+#include <assert.h>+#include <float.h>+#include <stdlib.h> // for qsort+#include <stdio.h> // for GGML_ASSERT++#define GROUP_MAX_EPS 1e-15f+#define GROUP_MAX_EPS_IQ3_XXS 1e-8f+#define GROUP_MAX_EPS_IQ2_S 1e-8f+#define GROUP_MAX_EPS_IQ1_M 1e-7f+#define GROUP_MAX_EPS_IQ1_S 1e-12f++#define UNUSED GGML_UNUSED++void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q1_0_ref(x, y, k);+}++void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q2_0_ref(x, y, k);+}++void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q4_0_ref(x, y, k);+}++void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q4_1_ref(x, y, k);+}++void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q5_0_ref(x, y, k);+}++void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q5_1_ref(x, y, k);+}++void quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q8_0_ref(x, y, k);+}++void quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q8_1_ref(x, y, k);+}++void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_mxfp4_ref(x, y, k);+}++void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_nvfp4_ref(x, y, k);+}++//+// 2-6 bit quantization in super-blocks+//++//========================- 2-bit (de)-quantization++void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ quantize_row_q2_K_ref(x, vy, k);+}++//========================= 3-bit (de)-quantization++void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ quantize_row_q3_K_ref(x, vy, k);+}++// ====================== 4-bit (de)-quantization++void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ assert(k % QK_K == 0);+ block_q4_K * GGML_RESTRICT y = vy;+ quantize_row_q4_K_ref(x, y, k);+}++// ====================== 5-bit (de)-quantization++void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ assert(k % QK_K == 0);+ block_q5_K * GGML_RESTRICT y = vy;+ quantize_row_q5_K_ref(x, y, k);+}++// ====================== 6-bit (de)-quantization++void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ assert(k % QK_K == 0);+ block_q6_K * GGML_RESTRICT y = vy;+ quantize_row_q6_K_ref(x, y, k);+}++// ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs)++void quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ assert(k % QK_K == 0);+ block_tq1_0 * GGML_RESTRICT y = vy;+ quantize_row_tq1_0_ref(x, y, k);+}++void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+ assert(k % QK_K == 0);+ block_tq2_0 * GGML_RESTRICT y = vy;+ quantize_row_tq2_0_ref(x, y, k);+}++//===================================== Q8_K ==============================================++void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q8_K_ref(x, y, k);+}++//===================================== Dot products =================================++void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK1_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q1_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d);++ float sumi = 0.0f;++ for (int k = 0; k < 4; k++) {+ const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k];+ const float d1 = GGML_CPU_FP16_TO_FP32(yb->d);+ int sumi_block = 0;++ const uint8_t * GGML_RESTRICT bits = &x[i].qs[k * 4];+ const int8_t * GGML_RESTRICT qy = yb->qs;++ for (int b = 0; b < 4; ++b, qy += 8) {+ const unsigned mask = bits[b];+ sumi_block += ((mask & 0x01) ? qy[0] : -qy[0])+ + ((mask & 0x02) ? qy[1] : -qy[1])+ + ((mask & 0x04) ? qy[2] : -qy[2])+ + ((mask & 0x08) ? qy[3] : -qy[3])+ + ((mask & 0x10) ? qy[4] : -qy[4])+ + ((mask & 0x20) ? qy[5] : -qy[5])+ + ((mask & 0x40) ? qy[6] : -qy[6])+ + ((mask & 0x80) ? qy[7] : -qy[7]);+ }++ sumi += d1 * sumi_block;+ }++ sumf += d0 * sumi;+ }++ *s = sumf;+}++void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK2_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ float sumf = 0.0f;++ for (int i = 0; i < nb; i++) {+ const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d);++ float sumi = 0.0f;++ // group 64: one Q2_0 block (64 weights) maps to two Q8_0 blocks (2 * 32 = 64)+ for (int k = 0; k < 2; k++) {+ const block_q8_0 * GGML_RESTRICT yb = &y[i * 2 + k];+ const float d1 = GGML_CPU_FP16_TO_FP32(yb->d);+ int sumi_block = 0;++ const uint8_t * GGML_RESTRICT qs = &x[i].qs[k * 8];+ const int8_t * GGML_RESTRICT qy = yb->qs;++ for (int b = 0; b < 8; ++b) {+ const uint8_t byte = qs[b];+ // Extract 4 two-bit values, map {0,1,2,3} -> {-1,0,1,2}+ sumi_block += ((int)((byte >> 0) & 3) - 1) * qy[b*4 + 0];+ sumi_block += ((int)((byte >> 2) & 3) - 1) * qy[b*4 + 1];+ sumi_block += ((int)((byte >> 4) & 3) - 1) * qy[b*4 + 2];+ sumi_block += ((int)((byte >> 6) & 3) - 1) * qy[b*4 + 3];+ }++ sumi += d1 * sumi_block;+ }++ sumf += d0 * sumi;+ }++ *s = sumf;+}++void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ int ib = 0;+ float sumf = 0;++ for (; ib < nb; ++ib) {+ int sumi0 = 0;+ int sumi1 = 0;++ for (int j = 0; j < qk/2; ++j) {+ const int v0 = (x[ib].qs[j] & 0x0F) - 8;+ const int v1 = (x[ib].qs[j] >> 4) - 8;++ sumi0 += (v0 * y[ib].qs[j]);+ sumi1 += (v1 * y[ib].qs[j + qk/2]);+ }++ int sumi = sumi0 + sumi1;+ sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);+ }++ *s = sumf;+}++// TODO: add WASM SIMD+void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_1;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_1 * GGML_RESTRICT x = vx;+ const block_q8_1 * GGML_RESTRICT y = vy;++ int ib = 0;+ float sumf = 0;++ for (; ib < nb; ++ib) {+ int sumi0 = 0;+ int sumi1 = 0;++ for (int j = 0; j < qk/2; ++j) {+ const int v0 = (x[ib].qs[j] & 0x0F);+ const int v1 = (x[ib].qs[j] >> 4);++ sumi0 += (v0 * y[ib].qs[j]);+ sumi1 += (v1 * y[ib].qs[j + qk/2]);+ }++ int sumi = sumi0 + sumi1;+ sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);+ }++ *s = sumf;+}++void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK_MXFP4 == 0);+ static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same");++ const block_mxfp4 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ const int nb = n / QK_MXFP4;++ int ib = 0;+ float sumf = 0;++ for (; ib < nb; ++ib) {+ const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_E8M0_TO_FP32_HALF(x[ib].e);++ int sumi1 = 0;+ int sumi2 = 0;+ for (int j = 0; j < QK_MXFP4/2; ++j) {+ sumi1 += y[ib].qs[j + 0] * kvalues_mxfp4[x[ib].qs[j] & 0xf];+ sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >> 4];+ }+ sumf += d * (sumi1 + sumi2);+ }+ *s = sumf;+}++// NVFP4: super-block of 64 elements = 4 sub-blocks of 16 = 2 q8_0 blocks+void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK_NVFP4 == 0);++ const block_nvfp4 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ const int nb = n / QK_NVFP4;++ float sumf = 0;++ for (int ib = 0; ib < nb; ++ib) {+ for (int s_idx = 0; s_idx < 4; ++s_idx) {+ const float d = ggml_ue4m3_to_fp32(x[ib].d[s_idx]);+ const int q8_block = s_idx / 2;+ const int q8_off = (s_idx % 2) * QK_NVFP4_SUB;+ const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8_block].d);++ int sumi_lo = 0, sumi_hi = 0;+ for (int j = 0; j < QK_NVFP4_SUB/2; ++j) {+ const uint8_t qv = x[ib].qs[s_idx*(QK_NVFP4_SUB/2) + j];+ sumi_lo += y[2*ib + q8_block].qs[q8_off + j + 0] * kvalues_mxfp4[qv & 0xf];+ sumi_hi += y[2*ib + q8_block].qs[q8_off + j + QK_NVFP4_SUB/2] * kvalues_mxfp4[qv >> 4];+ }++ sumf += dy * d * (sumi_lo + sumi_hi);+ }+ }+ *s = sumf;+}++void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ int ib = 0;+ float sumf = 0;++ assert(n % qk == 0);+ assert(qk == QK5_0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q5_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ for (; ib < nb; ++ib) {+ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));++ int sumi0 = 0;+ int sumi1 = 0;++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;+ const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));++ const int32_t x0 = (int8_t)(((x[ib].qs[j] & 0x0F) | xh_0) - 16);+ const int32_t x1 = (int8_t)(((x[ib].qs[j] >> 4) | xh_1) - 16);++ sumi0 += (x0 * y[ib].qs[j]);+ sumi1 += (x1 * y[ib].qs[j + qk/2]);+ }++ int sumi = sumi0 + sumi1;+ sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;+ }++ *s = sumf;+}++void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_1;+ const int nb = n / qk;++ int ib = 0;+ float sumf = 0;++ assert(n % qk == 0);+ assert(qk == QK5_1);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q5_1 * GGML_RESTRICT x = vx;+ const block_q8_1 * GGML_RESTRICT y = vy;++ for (; ib < nb; ++ib) {+ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));++ int sumi0 = 0;+ int sumi1 = 0;++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;++ const int32_t x0 = (x[ib].qs[j] & 0xF) | xh_0;+ const int32_t x1 = (x[ib].qs[j] >> 4) | xh_1;++ sumi0 += (x0 * y[ib].qs[j]);+ sumi1 += (x1 * y[ib].qs[j + qk/2]);+ }++ int sumi = sumi0 + sumi1;+ sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);+ }++ *s = sumf;+}++void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q8_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ int ib = 0;+ float sumf = 0;++ for (; ib < nb; ++ib) {+ int sumi = 0;++ for (int j = 0; j < qk; j++) {+ sumi += x[ib].qs[j]*y[ib].qs[j];+ }++ sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));+ }++ *s = sumf;+}++void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_tq1_0 * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243};++ float sumf = 0.0f;++ for (int i = 0; i < nb; ++i) {+ int sum = 0;++ for (size_t j = 0; j < sizeof(x->qs) - sizeof(x->qs) % 32; j += 32) {+ for (size_t l = 0; l < 5; ++l) {+ for (size_t m = 0; m < 32; ++m) {+ uint8_t q = x[i].qs[j + m] * pow3[l];+ uint16_t xi = ((uint16_t) q * 3) >> 8;+ sum += (xi - 1) * y[i].qs[j*5 + l*32 + m];+ }+ }+ }+ for (size_t j = sizeof(x->qs) - sizeof(x->qs) % 32; j < sizeof(x->qs); j += 16) {+ for (size_t l = 0; l < 5; ++l) {+ for (size_t m = 0; m < 16; ++m) {+ uint8_t q = x[i].qs[j + m] * pow3[l];+ uint16_t xi = ((uint16_t) q * 3) >> 8;+ sum += (xi - 1) * y[i].qs[j*5 + l*16 + m];+ }+ }+ }++ for (size_t l = 0; l < 4; ++l) {+ for (size_t j = 0; j < sizeof(x->qh); ++j) {+ uint8_t q = x[i].qh[j] * pow3[l];+ uint16_t xi = ((uint16_t) q * 3) >> 8;+ sum += (xi - 1) * y[i].qs[sizeof(x->qs)*5 + l*sizeof(x->qh) + j];+ }+ }++ sumf += (float) sum * (GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d);+ }++ *s = sumf;+}++void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_tq2_0 * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;+ float sumf = 0.0f;++ for (int i = 0; i < nb; ++i) {+ int32_t sumi = 0;++ for (size_t j = 0; j < sizeof(x->qs); j += 32) {+ for (size_t l = 0; l < 4; ++l) {+ for (size_t k = 0; k < 32; ++k) {+ sumi += y[i].qs[j*4 + l*32 + k] * (((x[i].qs[j + k] >> (l*2)) & 3) - 1);+ }+ }+ }++ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);++ sumf += (float) sumi * d;+ }++ *s = sumf;+}++void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ float sumf = 0;++ for (int i = 0; i < nb; ++i) {++ const uint8_t * q2 = x[i].qs;+ const int8_t * q8 = y[i].qs;+ const uint8_t * sc = x[i].scales;++ int summs = 0;+ for (int j = 0; j < 16; ++j) {+ summs += y[i].bsums[j] * (sc[j] >> 4);+ }++ const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);++ int isum = 0;+ int is = 0;+ int d;+ for (int k = 0; k < QK_K/128; ++k) {+ int shift = 0;+ for (int j = 0; j < 4; ++j) {+ d = sc[is++] & 0xF;+ int isuml = 0;+ for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);+ isum += d * isuml;+ d = sc[is++] & 0xF;+ isuml = 0;+ for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);+ isum += d * isuml;+ shift += 2;+ q8 += 32;+ }+ q2 += 32;+ }+ sumf += dall * isum - dmin * summs;+ }+ *s = sumf;+}++void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const uint32_t kmask1 = 0x03030303;+ const uint32_t kmask2 = 0x0f0f0f0f;++ const block_q3_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ // scalar version+ // This function is written like this so the compiler can manage to vectorize most of it+ // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the+ // manually vectorized version above. Every other version I tried would run at least 4 times slower.+ // The ideal situation would be if we could just write the code once, and the compiler would+ // automatically produce the best possible set of machine instructions, instead of us having to manually+ // write vectorized versions for AVX, ARM_NEON, etc.++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ uint32_t auxs[4];+ const int8_t * scales = (const int8_t*)auxs;++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * GGML_RESTRICT q3 = x[i].qs;+ const uint8_t * GGML_RESTRICT hm = x[i].hmask;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * GGML_RESTRICT a = aux8;+ uint8_t m = 1;+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ q3 += 32;+ }+ a = aux8;++ memcpy(auxs, x[i].scales, 12);+ uint32_t tmp = auxs[2];+ auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);+ auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);+ auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);+ auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);+ for (int j = 0; j < QK_K/16; ++j) {+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;+}++void ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;++ uint32_t utmp[4];++ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * GGML_RESTRICT q4 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * GGML_RESTRICT a = aux8;+ for (int j = 0; j < QK_K/64; ++j) {+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);+ a += 32;+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);+ a += 32; q4 += 32;+ }+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ int sumi = 0;+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];+ a = aux8;+ int is = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ int32_t scale = scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;+ sumf -= dmin * sumi;+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;+}++void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q5_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;++ uint32_t utmp[4];++ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * GGML_RESTRICT q4 = x[i].qs;+ const uint8_t * GGML_RESTRICT hm = x[i].qh;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * GGML_RESTRICT a = aux8;+ uint8_t m = 1;+ for (int j = 0; j < QK_K/64; ++j) {+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);+ a += 32; m <<= 1;+ q4 += 32;+ }+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ int sumi = 0;+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];+ a = aux8;+ int is = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ int32_t scale = scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;+ sumf -= dmin * sumi;+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;+}++void ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q6_K * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * GGML_RESTRICT q4 = x[i].ql;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * GGML_RESTRICT a = aux8;+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;+ a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;+ a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;+ a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;+ }+ a += 128;+ q4 += 64;+ qh += 32;+ }+ a = aux8;+ int is = 0;+ for (int j = 0; j < QK_K/16; ++j) {+ int scale = x[i].scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;+}++void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq2_xxs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ uint32_t aux32[2];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * GGML_RESTRICT q2 = x[i].qs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ memcpy(aux32, q2, 2*sizeof(uint32_t));+ q2 += 4;+ const uint32_t ls = 2*(aux32[1] >> 28) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);+ const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls;+ }+ sumf += d * bsum;+ }+ *s = 0.125f * sumf;+}++void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq2_xs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * GGML_RESTRICT q2 = x[i].qs;+ const uint8_t * GGML_RESTRICT sc = x[i].scales;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;+ const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 2; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls1;+ sumi = 0;+ for (int l = 2; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls2;+ q2 += 4;+ }+ sumf += d * bsum;+ }+ *s = 0.125f * sumf;+}++void ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq2_s * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ float sumf = 0;+ for (int i = 0; i < nb; i++) {++ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint8_t * qh = x[i].qh;+ const uint8_t * signs = qs + QK_K/8;++ int bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ int ls1 = 1 + 2*(x[i].scales[ib32] & 0xf);+ int ls2 = 1 + 2*(x[i].scales[ib32] >> 4);+ int sumi1 = 0, sumi2 = 0;+ for (int l = 0; l < 2; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));+ for (int j = 0; j < 8; ++j) {+ sumi1 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ for (int l = 2; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));+ for (int j = 0; j < 8; ++j) {+ sumi2 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += ls1 * sumi1 + ls2 * sumi2;+ qs += 4;+ signs += 4;+ }++ sumf += d * bsum;+ }++ *s = 0.125f * sumf;+}++void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq3_xxs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ uint32_t aux32;++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT q3 = x[i].qs;+ const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ memcpy(&aux32, gas, sizeof(uint32_t)); gas += sizeof(uint32_t);+ const uint32_t ls = 2*(aux32 >> 28) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*l+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*l+1]);+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];+ for (int j = 0; j < 4; ++j) {+ sumi += grid1[j] * q8[j+0] * (signs & kmask_iq2xs[j+0] ? -1 : 1);+ sumi += grid2[j] * q8[j+4] * (signs & kmask_iq2xs[j+4] ? -1 : 1);+ }+ q8 += 8;+ }+ q3 += 8;+ bsum += sumi * ls;+ }+ sumf += d * bsum;+ }+ *s = 0.25f * sumf;+}++void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq3_s * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * GGML_RESTRICT qs = x[i].qs;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const uint8_t * GGML_RESTRICT signs = x[i].signs;+ const int8_t * GGML_RESTRICT q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const uint32_t ls1 = 2*(x[i].scales[ib32/2] & 0xf) + 1;+ const uint32_t ls2 = 2*(x[i].scales[ib32/2] >> 4) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256)));+ for (int j = 0; j < 4; ++j) {+ sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);+ sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);+ }+ q8 += 8;+ }+ qs += 8;+ signs += 4;+ bsum += sumi * ls1;+ sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256)));+ for (int j = 0; j < 4; ++j) {+ sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);+ sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);+ }+ q8 += 8;+ }+ qs += 8;+ signs += 4;+ bsum += sumi * ls2;+ }+ sumf += d * bsum;+ }+ *s = sumf;+}++void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq1_s * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ float sumf = 0;+ for (int i = 0; i < nb; i++) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;++ int sumi = 0, sumi1 = 0;+ for (int ib = 0; ib < QK_K/32; ++ib) {+ const int ls = 2*((qh[ib] >> 12) & 7) + 1;+ const int delta = qh[ib] & 0x8000 ? -1 : 1;+ int lsum = 0;+ for (int l = 0; l < 4; ++l) {+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));+ for (int j = 0; j < 8; ++j) {+ lsum += q8[j] * grid[j];+ }+ q8 += 8;+ }+ sumi += ls * lsum;+ sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]);+ qs += 4;+ }++ sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);+ }++ *s = sumf;+}++void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq1_m * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ iq1m_scale_t scale;++ int sum1[2], sum2[2], delta[4];++ float sumf = 0;+ for (int i = 0; i < nb; i++) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint8_t * qh = x[i].qh;+ const uint16_t * sc = (const uint16_t *)x[i].scales;++ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);++ int sumi1 = 0, sumi2 = 0;+ for (int ib = 0; ib < QK_K/32; ++ib) {+ delta[0] = qh[0] & 0x08 ? -1 : 1;+ delta[1] = qh[0] & 0x80 ? -1 : 1;+ delta[2] = qh[1] & 0x08 ? -1 : 1;+ delta[3] = qh[1] & 0x80 ? -1 : 1;+ sum1[0] = sum1[1] = sum2[0] = sum2[1] = 0;+ for (int l = 0; l < 4; ++l) {+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((uint16_t)qh[l/2] << (8 - 4*(l%2))) & 0x700)));+ int lsum1 = 0, lsum2 = 0;+ for (int j = 0; j < 8; ++j) {+ lsum1 += q8[j] * grid[j];+ lsum2 += q8[j];+ }+ q8 += 8;+ sum1[l/2] += lsum1;+ sum2[l/2] += lsum2*delta[l];+ }++ const int ls1 = 2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1;+ const int ls2 = 2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1;++ sumi1 += sum1[0] * ls1 + sum1[1] * ls2;+ sumi2 += sum2[0] * ls1 + sum2[1] * ls2;+ qs += 4;+ qh += 2;+ }++ sumf += GGML_CPU_FP16_TO_FP32(scale.f16) * y[i].d * (sumi1 + IQ1M_DELTA * sumi2);+ }++ *s = sumf;+}++void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK4_NL == 0);+ static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same");++ const block_iq4_nl * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ const int nb = n / QK4_NL;++ int ib = 0;+ float sumf = 0;++ for (; ib < nb; ++ib) {+ const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < QK4_NL/2; ++j) {+ sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];+ sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];+ }+ sumf += d * (sumi1 + sumi2);+ }+ *s = sumf;+}++void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK_K == 0);++ const block_iq4_xs * GGML_RESTRICT x = vx;+ const block_q8_K * GGML_RESTRICT y = vy;++ const int nb = n / QK_K;++ float sumf = 0;+ for (int ibl = 0; ibl < nb; ++ibl) {+ const float d4d8 = GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d;+ uint16_t h = x[ibl].scales_h;+ const uint8_t * qs = x[ibl].qs;+ const int8_t * q8 = y[ibl].qs;+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ const uint8_t ls1 = (x[ibl].scales_l[ib/2] & 0xf) | ((h << 4) & 0x30);+ const uint8_t ls2 = (x[ibl].scales_l[ib/2] >> 4) | ((h << 2) & 0x30);+ h >>= 4;+ const float d1 = d4d8*(ls1 - 32);+ const float d2 = d4d8*(ls2 - 32);+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < 16; ++j) {+ sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];+ sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];+ }+ sumf += d1 * (sumi1 + sumi2);+ qs += 16;+ q8 += 32;+ sumi1 = sumi2 = 0;+ for (int j = 0; j < 16; ++j) {+ sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];+ sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];+ }+ sumf += d2 * (sumi1 + sumi2);+ qs += 16;+ q8 += 32;+ }+ }+ *s = sumf;+}++// ============================ 4-bit non-linear quants++void quantize_row_iq4_nl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ assert(k % QK4_NL == 0);+ quantize_row_iq4_nl_ref(x, y, k);+}++void quantize_row_iq4_xs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ quantize_iq4_xs(x, y, 1, k, NULL);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/quants.h b/python/freetoken/kernel/csrc/ggml/ggml-cpu/quants.hnew file mode 100644index 00000000..93ea7eef--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/quants.h@@ -0,0 +1,106 @@+#pragma once++#define GGML_COMMON_DECL_C+#include "ggml-common.h"++#include "ggml.h"++// GGML CPU internal header++#ifdef __cplusplus+extern "C" {+#endif++// Quantization+void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++void quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++// Dot product+void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++// Generic implementation+void quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);+void quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);+void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/simd-mappings.h b/python/freetoken/kernel/csrc/ggml/ggml-cpu/simd-mappings.hnew file mode 100644index 00000000..fb53ecdf--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/simd-mappings.h@@ -0,0 +1,1366 @@+#pragma once++#include "ggml-cpu-impl.h"++#ifdef __ARM_FEATURE_SVE+#include <arm_sve.h>+#endif // __ARM_FEATURE_SVE++#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)+// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:+//+// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/+//+#include <arm_neon.h>+#endif++#if defined(__riscv_v_intrinsic)+#include <riscv_vector.h>+#endif++#ifdef __cplusplus+extern "C" {+#endif++//+// simd mappings+//++// FP16 to FP32 conversion++// 16-bit float+// on Arm, we use __fp16, which requires the IEEE fp16 format: implied on+// AArch64, selected by -mfp16-format=ieee on 32 bit Arm, where the compiler+// may otherwise reject the type+// on x86, we use uint16_t+//+// for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616+// for MUSA compilers , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843+//+#if defined(__ARM_NEON) && defined(__ARM_FP16_FORMAT_IEEE) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__)+ #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x)+ #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x)++ #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)++ static inline float neon_compute_fp16_to_fp32(ggml_fp16_t h) {+ __fp16 tmp;+ memcpy(&tmp, &h, sizeof(ggml_fp16_t));+ return (float)tmp;+ }++ static inline ggml_fp16_t neon_compute_fp32_to_fp16(float f) {+ ggml_fp16_t res;+ __fp16 tmp = f;+ memcpy(&res, &tmp, sizeof(ggml_fp16_t));+ return res;+ }+#elif defined(__F16C__)+ #ifdef _MSC_VER+ #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))+ #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)+ #else+ #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)+ #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)+ #endif+#elif defined(__POWER9_VECTOR__)+ #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) power_compute_fp16_to_fp32(x)+ #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) power_compute_fp32_to_fp16(x)+ /* the inline asm below is about 12% faster than the lookup method */+ #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)+ #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)++ static inline float power_compute_fp16_to_fp32(ggml_fp16_t h) {+ float f;+ double d;+ __asm__(+ "mtfprd %0,%2\n"+ "xscvhpdp %0,%0\n"+ "frsp %1,%0\n" :+ /* temp */ "=d"(d),+ /* out */ "=f"(f):+ /* in */ "r"(h));+ return f;+ }++ static inline ggml_fp16_t power_compute_fp32_to_fp16(float f) {+ double d;+ ggml_fp16_t r;+ __asm__( /* xscvdphp can work on double or single precision */+ "xscvdphp %0,%2\n"+ "mffprd %1,%0\n" :+ /* temp */ "=d"(d),+ /* out */ "=r"(r):+ /* in */ "f"(f));+ return r;+ }+#elif defined(__riscv) && defined(__riscv_zfhmin)+ static inline float riscv_compute_fp16_to_fp32(ggml_fp16_t h) {+ _Float16 hf;+ memcpy(&hf, &h, sizeof(ggml_fp16_t));+ return hf;+ }++ static inline ggml_fp16_t riscv_compute_fp32_to_fp16(float f) {+ ggml_fp16_t res;+ _Float16 hf = (_Float16)f;+ memcpy(&res, &hf, sizeof(ggml_fp16_t));+ return res;+ }++ #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) riscv_compute_fp16_to_fp32(x)+ #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) riscv_compute_fp32_to_fp16(x)+ #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)+ #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)+#endif++// precomputed f32 table for f16 (256 KB)+// defined in ggml-cpu.c, initialized in ggml_cpu_init()+extern float ggml_table_f32_f16[1 << 16];++// precomputed f32 table for e8m0 half (1 KB)+// defined in ggml-cpu.c, initialized in ggml_cpu_init()+extern float ggml_table_f32_e8m0_half[1 << 8];++// precomputed f32 table for ue4m3 (1 KB)+// defined in ggml-cpu.c, initialized in ggml_cpu_init()+extern float ggml_table_f32_ue4m3[1 << 8];++// Use lookup table for E8M0 on x86 (faster than bit manipulation)+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+#define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)]+#else+#define GGML_CPU_E8M0_TO_FP32_HALF(x) GGML_E8M0_TO_FP32_HALF(x)+#endif++// Use lookup table for UE4M3 on x86 and ARM (faster than bit manipulation)+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__ARM_NEON)+#define GGML_CPU_UE4M3_TO_FP32(x) ggml_table_f32_ue4m3[(uint8_t)(x)]+#else+#define GGML_CPU_UE4M3_TO_FP32(x) ggml_ue4m3_to_fp32(x)+#endif++// Pre AVX512 CPUs (AVX2 with F16C, such as i5-8th gen or i7-4790)+// do not produce codegen as checks below don't test for __F16C__+// but this check does.+#if defined(__F16C__)+#define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)+#define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)+#endif++// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,+// so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON.+// This is also true for POWER9.+#if !defined(GGML_CPU_FP16_TO_FP32)+inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {+ uint16_t s;+ memcpy(&s, &f, sizeof(uint16_t));+ return ggml_table_f32_f16[s];+}++#define GGML_CPU_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x)+#endif++#if !defined(GGML_CPU_FP32_TO_FP16)+#define GGML_CPU_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)+#endif+++// we define a common set of C macros which map to specific intrinsics based on the current architecture+// we then implement the fundamental computation operations below using only these macros+// adding support for new architectures requires to define the corresponding SIMD macros+//+// GGML_F32_STEP / GGML_F16_STEP+// number of elements to process in a single step+//+// GGML_F32_EPR / GGML_F16_EPR+// number of elements to fit in a single register+//++#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_FMA)++#define GGML_SIMD++// F32 SVE+#define GGML_F32_EPR 8+#define DEFAULT_PG svptrue_b32()++#define GGML_F32xt svfloat32_t+#define GGML_F32xt_ZERO svdup_n_f32(0.0f)+#define GGML_F32xt_SET1(x) svdup_n_f32(x)+#define GGML_F32xt_LOAD_IMPL(pg, a) svld1_f32(pg, a)+#define GGML_F32xt_LOAD(a) GGML_F32xt_LOAD_IMPL(DEFAULT_PG, a)+#define GGML_F32xt_STORE_IMPL(pg, a, b) svst1_f32(pg, a, b)+#define GGML_F32xt_STORE(a, b) GGML_F32xt_STORE_IMPL(DEFAULT_PG, a, b)+#define GGML_F32xt_FMA_IMPL(pg, a, b, c) svmad_f32_m(pg, b, c, a)+#define GGML_F32xt_FMA(a, b, c) GGML_F32xt_FMA_IMPL(DEFAULT_PG, a, b, c)+#define GGML_F32xt_ADD_IMPL(pg, a, b) svadd_f32_m(pg, a, b)+#define GGML_F32xt_ADD(a, b) GGML_F32xt_ADD_IMPL(DEFAULT_PG, a, b)+#define GGML_F32xt_MUL_IMPL(pg, a, b) svmul_f32_m(pg, a, b)+#define GGML_F32xt_MUL(a, b) GGML_F32xt_MUL_IMPL(DEFAULT_PG, a, b)+#define GGML_F32xt_REDUCE_ONE_IMPL(pg, a) svaddv(pg, a)+#define GGML_F32xt_REDUCE_ONE(a) GGML_F32xt_REDUCE_ONE_IMPL(DEFAULT_PG, a)+#define GGML_F32xt_REDUCE_IMPL(pg, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) \+{ \+ sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum2); \+ sum3 = svadd_f32_m(DEFAULT_PG, sum3, sum4); \+ sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum6); \+ sum7 = svadd_f32_m(DEFAULT_PG, sum7, sum8); \+ sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum3); \+ sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum7); \+ sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum5); \+ (res) = (ggml_float) GGML_F32xt_REDUCE_ONE(sum1); \+}+#define GGML_F32xt_REDUCE(res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) \+ GGML_F32xt_REDUCE_IMPL(DEFAULT_PG, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8)++#define GGML_F32_VEC GGML_F32xt+#define GGML_F32_VEC_ZERO GGML_F32xt_ZERO+#define GGML_F32_VEC_SET1 GGML_F32xt_SET1+#define GGML_F32_VEC_LOAD GGML_F32xt_LOAD+#define GGML_F32_VEC_STORE GGML_F32xt_STORE+#define GGML_F32_VEC_FMA GGML_F32xt_FMA+#define GGML_F32_VEC_ADD GGML_F32xt_ADD+#define GGML_F32_VEC_MUL GGML_F32xt_MUL+#define GGML_F32_VEC_REDUCE GGML_F32xt_REDUCE++// F16 SVE+#define DEFAULT_PG32 svptrue_b32()+#define DEFAULT_PG16 svptrue_b16()++#define GGML_F32Cxt svfloat16_t+#define GGML_F32Cxt_ZERO svdup_n_f16(0.0f)+#define GGML_F32Cxt_SET1(x) svdup_n_f16(x)+#define GGML_F32Cxt_LOAD(p) svld1_f16(DEFAULT_PG16, (const __fp16 *)(p))+#define GGML_F32Cxt_STORE(dst_ptr, src_vec) svst1_f16(DEFAULT_PG16, (__fp16 *)(dst_ptr), (src_vec))++#define GGML_F32Cxt_FMA_IMPL(pg, a, b, c) svmad_f16_x(pg, b, c, a)+#define GGML_F32Cxt_FMA(a, b, c) GGML_F32Cxt_FMA_IMPL(DEFAULT_PG16, a, b, c)+#define GGML_F32Cxt_ADD_IMPL(pg, a, b) svadd_f16_x(pg, a, b)+#define GGML_F32Cxt_ADD(a, b) GGML_F32Cxt_ADD_IMPL(DEFAULT_PG16, a, b)+#define GGML_F32Cxt_MUL_IMPL(pg, a, b) svmul_f16_x(pg, a, b)+#define GGML_F32Cxt_MUL(a, b) GGML_F32Cxt_MUL_IMPL(DEFAULT_PG16, a, b)+#define GGML_F32Cxt_REDUCE GGML_F16xt_REDUCE_MIXED++#define GGML_F16x_VEC GGML_F32Cxt+#define GGML_F16x_VEC_ZERO GGML_F32Cxt_ZERO+#define GGML_F16x_VEC_SET1 GGML_F32Cxt_SET1+#define GGML_F16x_VEC_LOAD(p, i) GGML_F32Cxt_LOAD(p)+#define GGML_F16x_VEC_STORE(p, r, i) GGML_F32Cxt_STORE((__fp16 *)(p), r)+#define GGML_F16x_VEC_FMA GGML_F32Cxt_FMA+#define GGML_F16x_VEC_ADD GGML_F32Cxt_ADD+#define GGML_F16x_VEC_MUL GGML_F32Cxt_MUL+#define GGML_F16x_VEC_REDUCE GGML_F32Cxt_REDUCE++#define GGML_F16xt_REDUCE_ONE_IMPL(pg, a) svaddv_f16(pg, a)+#define GGML_F16xt_REDUCE_ONE(a) GGML_F16xt_REDUCE_ONE_IMPL(DEFAULT_PG16, a)++#define GGML_F16xt_REDUCE_MIXED_IMPL(pg16, res, sum1, sum2, sum3, sum4) \+{ \+ sum1 = svadd_f16_x(pg16, sum1, sum2); \+ sum3 = svadd_f16_x(pg16, sum3, sum4); \+ sum1 = svadd_f16_x(pg16, sum1, sum3); \+ __fp16 sum_f16 = svaddv_f16(pg16, sum1); \+ (res) = (ggml_float) sum_f16; \+}+#define GGML_F16xt_REDUCE_MIXED(res, sum1, sum2, sum3, sum4) \+ GGML_F16xt_REDUCE_MIXED_IMPL(DEFAULT_PG16, res, sum1, sum2, sum3, sum4)++// F16 NEON++#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)+ #define GGML_F16_STEP 32+ #define GGML_F16_EPR 8++ #define GGML_F16x8 float16x8_t+ #define GGML_F16x8_ZERO vdupq_n_f16(0.0f)+ #define GGML_F16x8_SET1(x) vdupq_n_f16(x)+ #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x))+ #define GGML_F16x8_STORE vst1q_f16+ #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)+ #define GGML_F16x8_ADD vaddq_f16+ #define GGML_F16x8_MUL vmulq_f16+ #define GGML_F16x8_REDUCE(res, x) \+ do { \+ int offset = GGML_F16_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \+ } \+ const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \+ const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \+ (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \+ } while (0)++ #define GGML_F16_VEC GGML_F16x8+ #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO+ #define GGML_F16_VEC_SET1 GGML_F16x8_SET1+ #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i])+ #define GGML_F16_VEC_FMA GGML_F16x8_FMA+ #define GGML_F16_VEC_ADD GGML_F16x8_ADD+ #define GGML_F16_VEC_MUL GGML_F16x8_MUL+ #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE+#else+ // if FP16 vector arithmetic is not supported, we use FP32 instead+ // and take advantage of the vcvt_ functions to convert to/from FP16++ #define GGML_F16_STEP 16+ #define GGML_F16_EPR 4++ #define GGML_F32Cx4 float32x4_t+ #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)+ #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)+ #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x)))+ #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))+ #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)+ #define GGML_F32Cx4_ADD vaddq_f32+ #define GGML_F32Cx4_MUL vmulq_f32+ #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++ #define GGML_F16_VEC GGML_F32Cx4+ #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+ #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+ #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i])+ #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+ #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+ #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+ #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE+#endif++#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) && defined(__ARM_FP16_FORMAT_IEEE)++#define GGML_SIMD++// F32 NEON++#define GGML_F32_STEP 16+#define GGML_F32_EPR 4++#define GGML_F32x4 float32x4_t+#define GGML_F32x4_ZERO vdupq_n_f32(0.0f)+#define GGML_F32x4_SET1(x) vdupq_n_f32(x)+#define GGML_F32x4_LOAD vld1q_f32+#define GGML_F32x4_STORE vst1q_f32+#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)+#define GGML_F32x4_ADD vaddq_f32+#define GGML_F32x4_MUL vmulq_f32+#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \+ } \+ (res) = (ggml_float) GGML_F32x4_REDUCE_ONE((x)[0]); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 NEON++#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)+ #define GGML_F16_STEP 32+ #define GGML_F16_EPR 8++ #define GGML_F16x8 float16x8_t+ #define GGML_F16x8_ZERO vdupq_n_f16(0.0f)+ #define GGML_F16x8_SET1(x) vdupq_n_f16(x)+ #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x))+ #define GGML_F16x8_STORE vst1q_f16+ #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)+ #define GGML_F16x8_ADD vaddq_f16+ #define GGML_F16x8_MUL vmulq_f16+ #define GGML_F16x8_REDUCE(res, x) \+ do { \+ int offset = GGML_F16_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \+ } \+ const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \+ const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \+ (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \+ } while (0)++ #define GGML_F16_VEC GGML_F16x8+ #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO+ #define GGML_F16_VEC_SET1 GGML_F16x8_SET1+ #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i])+ #define GGML_F16_VEC_FMA GGML_F16x8_FMA+ #define GGML_F16_VEC_ADD GGML_F16x8_ADD+ #define GGML_F16_VEC_MUL GGML_F16x8_MUL+ #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE+#else+ // if FP16 vector arithmetic is not supported, we use FP32 instead+ // and take advantage of the vcvt_ functions to convert to/from FP16++ #define GGML_F16_STEP 16+ #define GGML_F16_EPR 4++ #define GGML_F32Cx4 float32x4_t+ #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)+ #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)+ #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x)))+ #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))+ #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)+ #define GGML_F32Cx4_ADD vaddq_f32+ #define GGML_F32Cx4_MUL vmulq_f32+ #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++ #define GGML_F16_VEC GGML_F32Cx4+ #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+ #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+ #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i])+ #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+ #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+ #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+ #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE+#endif++#elif defined(__AVX512F__)++#define GGML_SIMD++// F32 AVX512++#define GGML_F32_STEP 64+#define GGML_F32_EPR 16++#define GGML_F32x16 __m512+#define GGML_F32x16_ZERO _mm512_setzero_ps()+#define GGML_F32x16_SET1(x) _mm512_set1_ps(x)+#define GGML_F32x16_LOAD _mm512_loadu_ps+#define GGML_F32x16_STORE _mm512_storeu_ps+// _mm512_fmadd_ps is defined in AVX512F so no guard is required+#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)+#define GGML_F32x16_ADD _mm512_add_ps+#define GGML_F32x16_MUL _mm512_mul_ps+#define GGML_F32x16_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ res = (ggml_float) _mm512_reduce_add_ps(x[0]); \+} while (0)++// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x16+#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x16_SET1+#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD+#define GGML_F32_VEC_STORE GGML_F32x16_STORE+#define GGML_F32_VEC_FMA GGML_F32x16_FMA+#define GGML_F32_VEC_ADD GGML_F32x16_ADD+#define GGML_F32_VEC_MUL GGML_F32x16_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE++// F16 AVX512++#if defined(__AVX512FP16__)++#define GGML_F16_STEP 128+#define GGML_F16_EPR 32++#define GGML_F16x32 __m512h+#define GGML_F16x32_ZERO _mm512_setzero_ph()+#define GGML_F16x32_SET1(x) _mm512_set1_ph(__extension__(_Float16)(x))+#define GGML_F16x32_LOAD(x) _mm512_loadu_ph(x)+#define GGML_F16x32_STORE(x, y) _mm512_storeu_ph(x, y)+#define GGML_F16x32_FMA(a, b, c) _mm512_fmadd_ph(b, c, a)+#define GGML_F16x32_ADD _mm512_add_ph+#define GGML_F16x32_MUL _mm512_mul_ph+#define GGML_F16x32_REDUCE(res, x) \+do { \+ int offset = GGML_F16_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ph(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ph(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ph(x[i], x[offset+i]); \+ } \+ res = (ggml_float) _mm512_reduce_add_ph(x[0]); \+} while (0)++#define GGML_F16_VEC GGML_F16x32+#define GGML_F16_VEC_ZERO GGML_F16x32_ZERO+#define GGML_F16_VEC_SET1 GGML_F16x32_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F16x32_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x32_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F16x32_FMA+#define GGML_F16_VEC_ADD GGML_F16x32_ADD+#define GGML_F16_VEC_MUL GGML_F16x32_MUL+#define GGML_F16_VEC_REDUCE GGML_F16x32_REDUCE++#else // Fallback FP16 <-> FP32++#define GGML_F16_STEP 64+#define GGML_F16_EPR 16++#define GGML_F32Cx16 __m512+#define GGML_F32Cx16_ZERO _mm512_setzero_ps()+#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x)++// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F+// so F16C guard isn't required+#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))+#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))++#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)+#define GGML_F32Cx16_ADD _mm512_add_ps+#define GGML_F32Cx16_MUL _mm512_mul_ps+#define GGML_F32Cx16_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ res = (ggml_float) _mm512_reduce_add_ps(x[0]); \+} while (0)++#define GGML_F16_VEC GGML_F32Cx16+#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL++#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE++#endif // __AVX512FP16__+#elif defined(__AVX__)++#define GGML_SIMD++// F32 AVX++#define GGML_F32_STEP 32+#define GGML_F32_EPR 8++#define GGML_F32x8 __m256+#define GGML_F32x8_ZERO _mm256_setzero_ps()+#define GGML_F32x8_SET1(x) _mm256_set1_ps(x)+#define GGML_F32x8_LOAD _mm256_loadu_ps+#define GGML_F32x8_STORE _mm256_storeu_ps+#if defined(__FMA__)+ #define GGML_F32x8_FMA(a, b, c) _mm256_fmadd_ps(b, c, a)+#else+ #define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)+#endif+#define GGML_F32x8_ADD _mm256_add_ps+#define GGML_F32x8_MUL _mm256_mul_ps+#define GGML_F32x8_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \+ _mm256_extractf128_ps(x[0], 1)); \+ const __m128 t1 = _mm_hadd_ps(t0, t0); \+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \+} while (0)+// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x8+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD+#define GGML_F32_VEC_STORE GGML_F32x8_STORE+#define GGML_F32_VEC_FMA GGML_F32x8_FMA+#define GGML_F32_VEC_ADD GGML_F32x8_ADD+#define GGML_F32_VEC_MUL GGML_F32x8_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE++// F16 AVX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 8++// F16 arithmetic is not supported by AVX, so we use F32 instead++#define GGML_F32Cx8 __m256+#define GGML_F32Cx8_ZERO _mm256_setzero_ps()+#define GGML_F32Cx8_SET1(x) _mm256_set1_ps(x)++#if defined(__F16C__)+// the _mm256_cvt intrinsics require F16C+#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))+#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))+#else+static inline __m256 __avx_f32cx8_load(const ggml_fp16_t * x) {+ float tmp[8];++ for (int i = 0; i < 8; i++) {+ tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);+ }++ return _mm256_loadu_ps(tmp);+}+static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {+ float arr[8];++ _mm256_storeu_ps(arr, y);++ for (int i = 0; i < 8; i++)+ x[i] = GGML_CPU_FP32_TO_FP16(arr[i]);+}+#define GGML_F32Cx8_LOAD(x) __avx_f32cx8_load(x)+#define GGML_F32Cx8_STORE(x, y) __avx_f32cx8_store(x, y)+#endif++#define GGML_F32Cx8_FMA GGML_F32x8_FMA+#define GGML_F32Cx8_ADD _mm256_add_ps+#define GGML_F32Cx8_MUL _mm256_mul_ps+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE++#define GGML_F16_VEC GGML_F32Cx8+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE++#elif defined(__POWER9_VECTOR__)++#define GGML_SIMD++// F32 POWER9++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 vector float+#define GGML_F32x4_ZERO {0.0f}+#define GGML_F32x4_SET1 vec_splats+#define GGML_F32x4_LOAD(p) vec_xl(0, p)+#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)+#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)+#define GGML_F32x4_ADD vec_add+#define GGML_F32x4_MUL vec_mul+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset+i]); \+ } \+ res = vec_extract(x[0], 0) + \+ vec_extract(x[0], 1) + \+ vec_extract(x[0], 2) + \+ vec_extract(x[0], 3); \+}+#define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \+{ \+ vector float v = vec_add(vec_add(s0, s1), \+ vec_add(s2, s3)); \+ v = vec_add(v, vec_sld(v, v, 8)); \+ v = vec_add(v, vec_sld(v, v, 4)); \+ res += (ggml_float) vec_extract(v, 0); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 POWER9+#define GGML_F16_STEP GGML_F32_STEP+#define GGML_F16_EPR GGML_F32_EPR+#define GGML_F16_VEC GGML_F32x4+#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32x4_SET1+#define GGML_F16_VEC_FMA GGML_F32x4_FMA+#define GGML_F16_VEC_ADD GGML_F32x4_ADD+#define GGML_F16_VEC_MUL GGML_F32x4_MUL+#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE+// Use vec_xl, not vec_ld, in case the load address is not aligned.+#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \+ vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \+ vec_extract_fp32_from_shortl(vec_xl(0, p))+static inline unsigned char ggml_endian_byte(int i) {+ uint16_t tmp_val = 1;+ return ((unsigned char *)&tmp_val)[i];+}+#define GGML_ENDIAN_BYTE(i) ggml_endian_byte(i)+#define GGML_F16_VEC_STORE(p, r, i) \+ if (i & 0x1) \+ vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)], \+ r[i - GGML_ENDIAN_BYTE(0)]), \+ 0, p - GGML_F16_EPR)++//BF16 POWER9+#define GGML_BF16_STEP 16+#define GGML_BF16_EPR 8++#define GGML_BF16x8 vector unsigned short+#define GGML_BF16x8_ZERO vec_splats((unsigned short)0)+#define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p))++#define GGML_BF16_VEC GGML_BF16x8+#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO+#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD+#if defined(__LITTLE_ENDIAN__)+#define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel(GGML_BF16_VEC_ZERO, (v)))+#define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh(GGML_BF16_VEC_ZERO, (v)))+#else+#define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel((v), GGML_BF16_VEC_ZERO))+#define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh((v), GGML_BF16_VEC_ZERO))+#endif+#define GGML_BF16_FMA_LO(acc, x, y) \+ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y))+#define GGML_BF16_FMA_HI(acc, x, y) \+ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))++#elif defined(__wasm_simd128__)++#define GGML_SIMD++// F32 WASM++#define GGML_F32_STEP 16+#define GGML_F32_EPR 4++#define GGML_F32x4 v128_t+#define GGML_F32x4_ZERO wasm_f32x4_splat(0.0f)+#define GGML_F32x4_SET1(x) wasm_f32x4_splat(x)+#define GGML_F32x4_LOAD wasm_v128_load+#define GGML_F32x4_STORE wasm_v128_store+#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)+#define GGML_F32x4_ADD wasm_f32x4_add+#define GGML_F32x4_MUL wasm_f32x4_mul+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ res = wasm_f32x4_extract_lane(x[0], 0) + \+ wasm_f32x4_extract_lane(x[0], 1) + \+ wasm_f32x4_extract_lane(x[0], 2) + \+ wasm_f32x4_extract_lane(x[0], 3); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 WASM++#define GGML_F16_STEP 16+#define GGML_F16_EPR 4++inline static v128_t __wasm_f16x4_load(const ggml_fp16_t * p) {+ float tmp[4];++ tmp[0] = GGML_CPU_FP16_TO_FP32(p[0]);+ tmp[1] = GGML_CPU_FP16_TO_FP32(p[1]);+ tmp[2] = GGML_CPU_FP16_TO_FP32(p[2]);+ tmp[3] = GGML_CPU_FP16_TO_FP32(p[3]);++ return wasm_v128_load(tmp);+}++inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {+ float tmp[4];++ wasm_v128_store(tmp, x);++ p[0] = GGML_CPU_FP32_TO_FP16(tmp[0]);+ p[1] = GGML_CPU_FP32_TO_FP16(tmp[1]);+ p[2] = GGML_CPU_FP32_TO_FP16(tmp[2]);+ p[3] = GGML_CPU_FP32_TO_FP16(tmp[3]);+}++#define GGML_F16x4 v128_t+#define GGML_F16x4_ZERO wasm_f32x4_splat(0.0f)+#define GGML_F16x4_SET1(x) wasm_f32x4_splat(x)+#define GGML_F16x4_LOAD(x) __wasm_f16x4_load(x)+#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)+#define GGML_F16x4_FMA GGML_F32x4_FMA+#define GGML_F16x4_ADD wasm_f32x4_add+#define GGML_F16x4_MUL wasm_f32x4_mul+#define GGML_F16x4_REDUCE(res, x) \+{ \+ int offset = GGML_F16_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ res = (ggml_float) (wasm_f32x4_extract_lane(x[0], 0) + \+ wasm_f32x4_extract_lane(x[0], 1) + \+ wasm_f32x4_extract_lane(x[0], 2) + \+ wasm_f32x4_extract_lane(x[0], 3)); \+}++#define GGML_F16_VEC GGML_F16x4+#define GGML_F16_VEC_ZERO GGML_F16x4_ZERO+#define GGML_F16_VEC_SET1 GGML_F16x4_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F16x4_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F16x4_FMA+#define GGML_F16_VEC_ADD GGML_F16x4_ADD+#define GGML_F16_VEC_MUL GGML_F16x4_MUL+#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE++#elif defined(__SSE3__)++#define GGML_SIMD++// F32 SSE++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 __m128+#define GGML_F32x4_ZERO _mm_setzero_ps()+#define GGML_F32x4_SET1(x) _mm_set1_ps(x)+#define GGML_F32x4_LOAD _mm_loadu_ps+#define GGML_F32x4_STORE _mm_storeu_ps+#if defined(__FMA__)+ // TODO: Does this work?+ #define GGML_F32x4_FMA(a, b, c) _mm_fmadd_ps(b, c, a)+#else+ #define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)+#endif+#define GGML_F32x4_ADD _mm_add_ps+#define GGML_F32x4_MUL _mm_mul_ps+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm_add_ps(x[i], x[offset+i]); \+ } \+ const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \+}+// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 SSE++#define GGML_F16_STEP 32+#define GGML_F16_EPR 4++static inline __m128 __sse_f16x4_load(const ggml_fp16_t * x) {+ float tmp[4];++ tmp[0] = GGML_CPU_FP16_TO_FP32(x[0]);+ tmp[1] = GGML_CPU_FP16_TO_FP32(x[1]);+ tmp[2] = GGML_CPU_FP16_TO_FP32(x[2]);+ tmp[3] = GGML_CPU_FP16_TO_FP32(x[3]);++ return _mm_loadu_ps(tmp);+}++static inline void __sse_f16x4_store(ggml_fp16_t * x, __m128 y) {+ float arr[4];++ _mm_storeu_ps(arr, y);++ x[0] = GGML_CPU_FP32_TO_FP16(arr[0]);+ x[1] = GGML_CPU_FP32_TO_FP16(arr[1]);+ x[2] = GGML_CPU_FP32_TO_FP16(arr[2]);+ x[3] = GGML_CPU_FP32_TO_FP16(arr[3]);+}++#define GGML_F32Cx4 __m128+#define GGML_F32Cx4_ZERO _mm_setzero_ps()+#define GGML_F32Cx4_SET1(x) _mm_set1_ps(x)+#define GGML_F32Cx4_LOAD(x) __sse_f16x4_load(x)+#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)+#define GGML_F32Cx4_FMA GGML_F32x4_FMA+#define GGML_F32Cx4_ADD _mm_add_ps+#define GGML_F32Cx4_MUL _mm_mul_ps+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++#define GGML_F16_VEC GGML_F32Cx4+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE++#elif defined(__loongarch_asx)++#define GGML_SIMD++// F32 LASX+#define GGML_F32_STEP 32+#define GGML_F32_EPR 8++#define GGML_F32x8 __m256+#define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0)+#define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))+#define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0)+#define GGML_F32x8_STORE(x,y) __lasx_xvst((y), (x), 0)+#define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a)+#define GGML_F32x8_ADD __lasx_xvfadd_s+#define GGML_F32x8_MUL __lasx_xvfmul_s+#define GGML_F32x8_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ float *tmp_p = (float *)&x[0]; \+ res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \+} while (0)+// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x8+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD+#define GGML_F32_VEC_STORE GGML_F32x8_STORE+#define GGML_F32_VEC_FMA GGML_F32x8_FMA+#define GGML_F32_VEC_ADD GGML_F32x8_ADD+#define GGML_F32_VEC_MUL GGML_F32x8_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE++// F16 LASX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 8++// F16 arithmetic is not supported by LASX, so we use F32 instead++#define GGML_F32Cx8 __m256+#define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0)+#define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))++static inline __m256 __lasx_f32cx8_load(const ggml_fp16_t * x) {+ __m256i a;+ memcpy(&a, x, sizeof(ggml_fp16_t) * 8);+ a = __lasx_xvpermi_d(a, 0 | (1 << 4));+ return __lasx_xvfcvtl_s_h(a);+}++static inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) {+ __m256i a = __lasx_xvfcvt_h_s(y, y);+ a = __lasx_xvpermi_d(a, 0 | (2 << 2));+ memcpy(x, &a, sizeof(ggml_fp16_t) * 8);+}+#define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x)+#define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y)++#define GGML_F32Cx8_FMA GGML_F32x8_FMA+#define GGML_F32Cx8_ADD __lasx_xvfadd_s+#define GGML_F32Cx8_MUL __lasx_xvfmul_s+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE++#define GGML_F16_VEC GGML_F32Cx8+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE++#elif defined(__loongarch_sx)++#define GGML_SIMD++// F32 LSX++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 __m128+#define GGML_F32x4_ZERO (__m128)__lsx_vldi(0)+#define GGML_F32x4_SET1(x) (__m128)__lsx_vreplfr2vr_s((x))+#define GGML_F32x4_LOAD(x) (__m128)__lsx_vld((x), 0)+#define GGML_F32x4_STORE(x, y) __lsx_vst(y, x, 0)+#define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a)+#define GGML_F32x4_ADD __lsx_vfadd_s+#define GGML_F32x4_MUL __lsx_vfmul_s++#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ __m128i t0 = __lsx_vpickev_w((__m128i)x[0], (__m128i)x[0]); \+ __m128i t1 = __lsx_vpickod_w((__m128i)x[0], (__m128i)x[0]); \+ __m128 t2 = __lsx_vfadd_s((__m128)t0, (__m128)t1); \+ __m128i t3 = __lsx_vpickev_w((__m128i)t2, (__m128i)t2); \+ __m128i t4 = __lsx_vpickod_w((__m128i)t2, (__m128i)t2); \+ __m128 t5 = __lsx_vfadd_s((__m128)t3, (__m128)t4); \+ res = (ggml_float) ((v4f32)t5)[0]; \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 LSX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 4++static inline __m128 __lsx_f16x4_load(const ggml_fp16_t * x) {+ return __lsx_vfcvtl_s_h(__lsx_vld((const void *)x, 0));+}++static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) {+ __m128i a = __lsx_vfcvt_h_s(y, y);+ memcpy(x, &a, sizeof(ggml_fp16_t) * 4);+}++#define GGML_F32Cx4 __m128+#define GGML_F32Cx4_ZERO (__m128)__lsx_vldi(0)+#define GGML_F32Cx4_SET1(x) (__m128)__lsx_vreplfr2vr_s((x))+#define GGML_F32Cx4_LOAD(x) (__m128)__lsx_f16x4_load(x)+#define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y)+#define GGML_F32Cx4_FMA GGML_F32x4_FMA+#define GGML_F32Cx4_ADD __lsx_vfadd_s+#define GGML_F32Cx4_MUL __lsx_vfmul_s+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++#define GGML_F16_VEC GGML_F32Cx4+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE++#elif defined(__VXE__) || defined(__VXE2__)++#define GGML_SIMD++// F32 s390x++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 float32x4_t+#define GGML_F32x4_ZERO vec_splats(0.0f)+#define GGML_F32x4_SET1 vec_splats+#define GGML_F32x4_LOAD(p) vec_xl(0, p)+#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)+#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)+#define GGML_F32x4_ADD vec_add+#define GGML_F32x4_MUL vec_mul+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset + i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset + i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset + i]); \+ } \+ float32x4_t tmp = x[0] + vec_reve(x[0]); \+ res = tmp[0] + tmp[1]; \+}+#define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \+{ \+ float32x4_t v = vec_add(vec_add(s0, s1), \+ vec_add(s2, s3)); \+ v = vec_add(v, vec_sld(v, v, 8)); \+ v = vec_add(v, vec_sld(v, v, 4)); \+ res += (ggml_float)vec_extract(v, 0); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 s390x+#define GGML_F16_STEP GGML_F32_STEP+#define GGML_F16_EPR GGML_F32_EPR++static inline uint32x4_t __lzs_f32cx4_to_f16(float32x4_t v_f) {+ float32x4_t v_base = vec_mul(vec_mul(vec_abs(v_f), vec_splats(0x1.0p+112f)), vec_splats(0x1.0p-110f));++ const uint32x4_t v_w = (uint32x4_t)v_f;+ const uint32x4_t v_shl1_w = vec_add(v_w, v_w);+ const uint32x4_t v_sign = vec_and(v_w, vec_splats(UINT32_C(0x80000000)));+ const uint32x4_t v_bias = vec_max(vec_and(v_shl1_w, vec_splats(UINT32_C(0xFF000000))), vec_splats(UINT32_C(0x71000000)));++ v_base = vec_add((float32x4_t)vec_add(vec_sr(v_bias, 1), vec_splats(UINT32_C(0x07800000))), v_base);++ const uint32x4_t v_bits = (uint32x4_t)v_base;+ const uint32x4_t v_nonsign = vec_add(vec_and(vec_sr(v_bits, 13), vec_splats(UINT32_C(0x00007C00))),+ vec_and(v_bits, vec_splats(UINT32_C(0x00000FFF))));+ const uint32x4_t v_is_nan = (uint32x4_t)vec_cmpgt(v_shl1_w, vec_splats(UINT32_C(0xFF000000)));++ return vec_or(vec_sr(v_sign, 16), vec_sel(v_nonsign, vec_splats(UINT32_C(0x7E00)), v_is_nan));+}++static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) {+ float tmp[4];++ for (int i = 0; i < 4; i++) {+ tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);+ }++ // note: keep type-cast here to prevent compiler bugs+ // see: https://github.com/ggml-org/llama.cpp/issues/12846+ return vec_xl(0, (const float *)(tmp));+}++static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) {+ const uint32x4_t v_h = __lzs_f32cx4_to_f16(v_y);+ const uint64_t tmp = ((uint64x2_t)vec_pack(v_h, v_h))[0];+ memcpy(x, &tmp, sizeof(tmp));+}++#define GGML_F16_VEC GGML_F32x4+#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32x4_SET1+#define GGML_F16_VEC_LOAD(p, i) __lzs_f16cx4_load(p)+#define GGML_F16_VEC_STORE(p, r, i) __lzs_f16cx4_store(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32x4_FMA+#define GGML_F16_VEC_ADD GGML_F32x4_ADD+#define GGML_F16_VEC_MUL GGML_F32x4_MUL+#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE++// BF16 s390x+#define GGML_BF16_STEP 16+#define GGML_BF16_EPR 8++#define GGML_BF16x8 __vector unsigned short+#define GGML_BF16x8_ZERO vec_splats((unsigned short)0)+#define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p))++#define GGML_BF16_VEC GGML_BF16x8+#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO+#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD+#define GGML_BF16_TO_F32_LO(v) ((float32x4_t) vec_mergel((v), GGML_BF16_VEC_ZERO))+#define GGML_BF16_TO_F32_HI(v) ((float32x4_t) vec_mergeh((v), GGML_BF16_VEC_ZERO))+#define GGML_BF16_FMA_LO(acc, x, y) \+ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y))+#define GGML_BF16_FMA_HI(acc, x, y) \+ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))++#elif defined(__riscv_v_intrinsic)++// compatible with vlen >= 128++#define GGML_SIMD++// F32++#define GGML_F32_STEP 16+#define GGML_F32_EPR 4++#define GGML_F32x4 vfloat32m1_t+#define GGML_F32x4_ZERO __riscv_vfmv_v_f_f32m1(0.0f, GGML_F32_EPR)+#define GGML_F32x4_SET1(x) __riscv_vfmv_v_f_f32m1(x, GGML_F32_EPR)+#define GGML_F32x4_LOAD(x) __riscv_vle32_v_f32m1(x, GGML_F32_EPR)+#define GGML_F32x4_STORE(b, v) __riscv_vse32_v_f32m1(b, v, GGML_F32_EPR)+#define GGML_F32x4_FMA(a, b, c) __riscv_vfmacc_vv_f32m1(a, b, c, GGML_F32_EPR)+#define GGML_F32x4_ADD(a, b) __riscv_vfadd_vv_f32m1(a, b, GGML_F32_EPR)+#define GGML_F32x4_MUL(a, b) __riscv_vfmul_vv_f32m1(a, b, GGML_F32_EPR)++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++#endif++// GGML_F32_ARR / GGML_F16_ARR+// number of registers to use per step+#ifdef GGML_SIMD+#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR)+#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)+#endif++// GGML_F16_DOT_*+// like GGML_F16_* but for dot products which need F32 accumulation on AVX512-FP16++#if defined(__AVX512FP16__)++#define GGML_F16_DOT_STEP GGML_F32_STEP+#define GGML_F16_DOT_EPR GGML_F32_EPR+#define GGML_F16_DOT_ARR GGML_F32_ARR+#define GGML_F16_DOT_VEC GGML_F32x16+#define GGML_F16_DOT_VEC_ZERO GGML_F32x16_ZERO+#define GGML_F16_DOT_VEC_LOAD(p, i) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)))+#define GGML_F16_DOT_VEC_FMA GGML_F32x16_FMA+#define GGML_F16_DOT_VEC_REDUCE GGML_F32x16_REDUCE++#else++#define GGML_F16_DOT_STEP GGML_F16_STEP+#define GGML_F16_DOT_EPR GGML_F16_EPR+#define GGML_F16_DOT_ARR GGML_F16_ARR+#define GGML_F16_DOT_VEC GGML_F16_VEC+#define GGML_F16_DOT_VEC_ZERO GGML_F16_VEC_ZERO+#define GGML_F16_DOT_VEC_LOAD GGML_F16_VEC_LOAD+#define GGML_F16_DOT_VEC_FMA GGML_F16_VEC_FMA+#define GGML_F16_DOT_VEC_REDUCE GGML_F16_VEC_REDUCE++#endif // defined(__AVX512FP16__)++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/vec.cpp b/python/freetoken/kernel/csrc/ggml/ggml-cpu/vec.cppnew file mode 100644index 00000000..3918a6f4--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/vec.cpp@@ -0,0 +1,613 @@+#include "vec.h"++#include <cassert>++// precomputed gelu table for f16 (128 KB)+ggml_fp16_t ggml_table_gelu_f16[1 << 16];++// precomputed quick gelu table for f16 (128 KB)+ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];++void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) {+ assert(nrc == 1);+ GGML_UNUSED(nrc);+ GGML_UNUSED(bx);+ GGML_UNUSED(by);+ GGML_UNUSED(bs);++#if defined(GGML_SIMD)+ float sumf = 0.0f;++ #if defined(__ARM_FEATURE_SVE)+ const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;+ const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16+ const int ggml_f32_step = 8 * ggml_f32_epr; // choose 8 SVE registers++ const int np = (n & ~(ggml_f32_step - 1));+ svfloat32_t sum1 = svdup_n_f32(0.0f);+ svfloat32_t sum2 = svdup_n_f32(0.0f);+ svfloat32_t sum3 = svdup_n_f32(0.0f);+ svfloat32_t sum4 = svdup_n_f32(0.0f);+ svfloat32_t sum5 = svdup_n_f32(0.0f);+ svfloat32_t sum6 = svdup_n_f32(0.0f);+ svfloat32_t sum7 = svdup_n_f32(0.0f);+ svfloat32_t sum8 = svdup_n_f32(0.0f);+ svfloat32_t ax1,ax2,ax3,ax4,ax5,ax6,ax7,ax8;+ svfloat32_t ay1,ay2,ay3,ay4,ay5,ay6,ay7,ay8;+ for (int i = 0; i < np; i += ggml_f32_step) {+ ax1 = GGML_F32_VEC_LOAD(x + i);+ ay1 = GGML_F32_VEC_LOAD(y + i);+ sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);++ ax2 = GGML_F32_VEC_LOAD(x + i + 1*ggml_f32_epr);+ ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);+ sum2 = GGML_F32_VEC_FMA(sum2, ax2, ay2);++ ax3 = GGML_F32_VEC_LOAD(x + i + 2*ggml_f32_epr);+ ay3 = GGML_F32_VEC_LOAD(y + i + 2*ggml_f32_epr);+ sum3 = GGML_F32_VEC_FMA(sum3, ax3, ay3);++ ax4 = GGML_F32_VEC_LOAD(x + i + 3*ggml_f32_epr);+ ay4 = GGML_F32_VEC_LOAD(y + i + 3*ggml_f32_epr);+ sum4 = GGML_F32_VEC_FMA(sum4, ax4, ay4);++ ax5 = GGML_F32_VEC_LOAD(x + i + 4*ggml_f32_epr);+ ay5 = GGML_F32_VEC_LOAD(y + i + 4*ggml_f32_epr);+ sum5 = GGML_F32_VEC_FMA(sum5, ax5, ay5);++ ax6 = GGML_F32_VEC_LOAD(x + i + 5*ggml_f32_epr);+ ay6 = GGML_F32_VEC_LOAD(y + i + 5*ggml_f32_epr);+ sum6 = GGML_F32_VEC_FMA(sum6, ax6, ay6);++ ax7 = GGML_F32_VEC_LOAD(x + i + 6*ggml_f32_epr);+ ay7 = GGML_F32_VEC_LOAD(y + i + 6*ggml_f32_epr);+ sum7 = GGML_F32_VEC_FMA(sum7, ax7, ay7);++ ax8 = GGML_F32_VEC_LOAD(x + i + 7*ggml_f32_epr);+ ay8 = GGML_F32_VEC_LOAD(y + i + 7*ggml_f32_epr);+ sum8 = GGML_F32_VEC_FMA(sum8, ax8, ay8);+ }+ // leftovers+ // Since 8 unrolls are done in above loop, leftovers lie in range [0, ggml_f32_step] which is handled in below loop+ const int np2 = (n & ~(ggml_f32_epr - 1));+ for (int i = np; i < np2; i += ggml_f32_epr) {+ ax1 = GGML_F32_VEC_LOAD(x + i);+ ay1 = GGML_F32_VEC_LOAD(y + i);+ sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);+ }+ // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmla on available elements only+ if (np2 < n) {+ svbool_t pg = svwhilelt_b32(np2, n);+ ax1 = svld1_f32(pg, x + np2);+ ay1 = svld1_f32(pg, y + np2);+ sum1 = svmla_f32_m(pg, sum1, ax1, ay1);+ }+ // reduce sum1,sum2 to sum1+ GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8);+ #elif defined(__riscv_v_intrinsic)+ int vl = __riscv_vsetvlmax_e32m8();+ vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);+ vfloat32m8_t vsum;+ vfloat32m8_t ax;+ vfloat32m8_t ay;+ vsum = __riscv_vfmv_v_f_f32m8_tu(vsum, 0.0f, vl);+ for (int i = 0; i < n; i += vl) {+ vl = __riscv_vsetvl_e32m8(n - i);+ ax = __riscv_vle32_v_f32m8_tu(ax, &x[i], vl);+ ay = __riscv_vle32_v_f32m8_tu(ay, &y[i], vl);+ vsum = __riscv_vfmacc_vv_f32m8_tu(vsum, ax, ay, vl);+ }+ vl = __riscv_vsetvlmax_e32m8();+ vs = __riscv_vfredusum_vs_f32m8_f32m1(vsum, vs, vl);+ sumf += __riscv_vfmv_f_s_f32m1_f32(vs);+ #else+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };++ GGML_F32_VEC ax[GGML_F32_ARR];+ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);++ sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);+ }+ }++ // reduce sum0..sum3 to sum0+ GGML_F32_VEC_REDUCE(sumf, sum);++ // leftovers+ for (int i = np; i < n; ++i) {+ sumf += x[i]*y[i];+ }+ #endif+#else+ // scalar+ ggml_float sumf = 0.0;+ for (int i = 0; i < n; ++i) {+ sumf += (ggml_float)(x[i]*y[i]);+ }+#endif++ *s = sumf;+}++void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc) {+ assert(nrc == 1);+ GGML_UNUSED(nrc);+ GGML_UNUSED(bx);+ GGML_UNUSED(by);+ GGML_UNUSED(bs);+ int i = 0;+ ggml_float sumf = 0;++#if defined(__AVX512BF16__)+ __m512 c1 = _mm512_setzero_ps();+ __m512 c2 = _mm512_setzero_ps();+ for (; i + 64 <= n; i += 64) {+ c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))),+ m512bh(_mm512_loadu_si512((y + i))));+ c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))),+ m512bh(_mm512_loadu_si512((y + i + 32))));+ }+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);++#elif defined(__AVX512F__)+#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))+ __m512 c1 = _mm512_setzero_ps();+ __m512 c2 = _mm512_setzero_ps();+ for (; i + 32 <= n; i += 32) {+ c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);+ c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);+ }+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);++#undef LOAD+#elif defined(__AVX2__) || defined(__AVX__)+#if defined(__AVX2__)+#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))+#else+#define LOAD(p) _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16)), (_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_bsrli_si128(_mm_loadu_si128((const __m128i *)(p)), 8)), 16)), 1))+#endif+ __m256 c1 = _mm256_setzero_ps();+ __m256 c2 = _mm256_setzero_ps();+ __m256 c3 = _mm256_setzero_ps();+ __m256 c4 = _mm256_setzero_ps();+ for (; i + 32 <= n; i += 32) {+ c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);+ c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);+ c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);+ c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);+ }+ __m128 g;+ c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),+ _mm256_add_ps(c2, c4));+ g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),+ _mm256_castps256_ps128(c1));+ g = _mm_add_ps(g, _mm_movehl_ps(g, g));+ g = _mm_add_ss(g, _mm_movehdup_ps(g));+ sumf += (ggml_float)_mm_cvtss_f32(g);++#undef LOAD+#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma)+ size_t vl = __riscv_vsetvlmax_e32m4();++ // initialize accumulators to all zeroes+ vfloat32m4_t vsum0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);+ vfloat32m4_t vsum1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);++ // calculate step size+ const size_t epr = __riscv_vsetvlmax_e16m2();+ const size_t step = epr * 2;+ const int np = (n & ~(step - 1));++ // unroll by 2+ for (; i < np; i += step) {+ vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], epr);+ vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], epr);+ vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, epr);+ __asm__ __volatile__ ("" ::: "memory");++ vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i + epr], epr);+ vbfloat16m2_t ay1 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i + epr], epr);+ vsum1 = __riscv_vfwmaccbf16_vv_f32m4(vsum1, ax1, ay1, epr);+ __asm__ __volatile__ ("" ::: "memory");+ }++ // accumulate in 1 register+ vsum0 = __riscv_vfadd_vv_f32m4(vsum0, vsum1, vl);++ // leftovers+ for (i = np; i < n; i += vl) {+ vl = __riscv_vsetvl_e16m2(n - i);+ vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], vl);+ vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], vl);+ vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, vl);+ }++ // reduce+ vl = __riscv_vsetvlmax_e32m4();+ vfloat32m1_t redsum = __riscv_vfredusum_vs_f32m4_f32m1(vsum0, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);+ sumf += __riscv_vfmv_f_s_f32m1_f32(redsum);++#elif defined(__POWER9_VECTOR__) || defined(__VXE__) || defined(__VXE2__)+ const int np = (n & ~(GGML_BF16_STEP - 1));+ if (np > 0) {+ GGML_F32_VEC sum[4] = {GGML_F32_VEC_ZERO};+ for (; i < np; i += GGML_BF16_STEP) {+ GGML_BF16_VEC vx0 = GGML_BF16_VEC_LOAD(x + i);+ GGML_BF16_VEC vx1 = GGML_BF16_VEC_LOAD(x + i + 8);+ GGML_BF16_VEC vy0 = GGML_BF16_VEC_LOAD(y + i);+ GGML_BF16_VEC vy1 = GGML_BF16_VEC_LOAD(y + i + 8);+ GGML_BF16_FMA_LO(sum[0], vx0, vy0);+ GGML_BF16_FMA_HI(sum[1], vx0, vy0);+ GGML_BF16_FMA_LO(sum[2], vx1, vy1);+ GGML_BF16_FMA_HI(sum[3], vx1, vy1);+ }+ GGML_F32x4_REDUCE_4(sumf, sum[0], sum[1], sum[2], sum[3]);+ }+#endif++ for (; i < n; ++i) {+ sumf += (ggml_float)(GGML_BF16_TO_FP32(x[i]) *+ GGML_BF16_TO_FP32(y[i]));+ }+ *s = sumf;+}++void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc) {+ assert(nrc == 1);+ GGML_UNUSED(nrc);+ GGML_UNUSED(bx);+ GGML_UNUSED(by);+ GGML_UNUSED(bs);++ ggml_float sumf = 0.0;+++#if defined(GGML_SIMD)+ #if defined(__ARM_FEATURE_SVE)+ const int ggml_f16_epr = svcnth();+ const int ggml_f16_step = 8 * ggml_f16_epr;+ const int np = n - (n % ggml_f16_step);+ const int np2 = n - (n % ggml_f16_epr);++ svfloat32_t sum1_lo = svdup_n_f32(0.0f);+ svfloat32_t sum1_hi = svdup_n_f32(0.0f);+ svfloat32_t sum2_lo = svdup_n_f32(0.0f);+ svfloat32_t sum2_hi = svdup_n_f32(0.0f);+ svfloat32_t sum3_lo = svdup_n_f32(0.0f);+ svfloat32_t sum3_hi = svdup_n_f32(0.0f);+ svfloat32_t sum4_lo = svdup_n_f32(0.0f);+ svfloat32_t sum4_hi = svdup_n_f32(0.0f);++ for (int i = 0; i < np; i += ggml_f16_step) {+ ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0), GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0));+ ggml_sve_f16_fma_widened(&sum2_lo, &sum2_hi, GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1), GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1));+ ggml_sve_f16_fma_widened(&sum3_lo, &sum3_hi, GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2), GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2));+ ggml_sve_f16_fma_widened(&sum4_lo, &sum4_hi, GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3), GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3));+ ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4), GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4));+ ggml_sve_f16_fma_widened(&sum2_lo, &sum2_hi, GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5), GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5));+ ggml_sve_f16_fma_widened(&sum3_lo, &sum3_hi, GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6), GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6));+ ggml_sve_f16_fma_widened(&sum4_lo, &sum4_hi, GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7), GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7));+ }++ for (int i = np; i < np2; i += ggml_f16_epr) {+ ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i, 0), GGML_F16x_VEC_LOAD(y + i, 0));+ }++ if (np2 < n) {+ const svbool_t pg = svwhilelt_b16(np2, n);+ const svfloat16_t rx = svld1_f16(pg, (const __fp16 *)(x + np2));+ const svfloat16_t ry = svld1_f16(pg, (const __fp16 *)(y + np2));++ ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, rx, ry);+ }++ sum1_lo = svadd_f32_m(DEFAULT_PG32, sum1_lo, sum2_lo);+ sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum2_hi);+ sum3_lo = svadd_f32_m(DEFAULT_PG32, sum3_lo, sum4_lo);+ sum3_hi = svadd_f32_m(DEFAULT_PG32, sum3_hi, sum4_hi);+ sum1_lo = svadd_f32_m(DEFAULT_PG32, sum1_lo, sum3_lo);+ sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum3_hi);++ sumf = ggml_sve_sum_f32x2(sum1_lo, sum1_hi);+ #elif defined(__riscv_v_intrinsic)+ #if defined(__riscv_zvfh)+ int vl = __riscv_vsetvlmax_e32m2();+ vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);+ vfloat32m2_t vsum;+ vfloat16m1_t ax;+ vfloat16m1_t ay;+ vsum = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vmv_v_x_u32m2(0, vl));+ for (int i = 0; i < n; i += vl) {+ vl = __riscv_vsetvl_e16m1(n - i);+ ax = __riscv_vle16_v_f16m1_tu(ax, (const _Float16 *)&x[i], vl);+ ay = __riscv_vle16_v_f16m1_tu(ay, (const _Float16 *)&y[i], vl);+ vsum = __riscv_vfwmacc_vv_f32m2_tu(vsum, ax, ay, vl);+ }+ vl = __riscv_vsetvlmax_e32m1();+ vfloat32m1_t ac0 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(vsum, 0), __riscv_vget_v_f32m2_f32m1(vsum, 1), vl);+ vs = __riscv_vfredusum_vs_f32m1_f32m1(ac0, vs, vl);+ sumf += __riscv_vfmv_f_s_f32m1_f32(vs);+ #else+ for (int i = 0; i < n; ++i) {+ sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));+ }+ #endif // __riscv_zvfh+ #else+ const int np = (n & ~(GGML_F16_DOT_STEP - 1));++ GGML_F16_DOT_VEC sum[GGML_F16_DOT_ARR] = { GGML_F16_DOT_VEC_ZERO };++ GGML_F16_DOT_VEC ax[GGML_F16_DOT_ARR];+ GGML_F16_DOT_VEC ay[GGML_F16_DOT_ARR];++ for (int i = 0; i < np; i += GGML_F16_DOT_STEP) {+ for (int j = 0; j < GGML_F16_DOT_ARR; j++) {+ ax[j] = GGML_F16_DOT_VEC_LOAD(x + i + j*GGML_F16_DOT_EPR, j);+ ay[j] = GGML_F16_DOT_VEC_LOAD(y + i + j*GGML_F16_DOT_EPR, j);++ sum[j] = GGML_F16_DOT_VEC_FMA(sum[j], ax[j], ay[j]);+ }+ }++ // reduce sum0..sum3 to sum0+ GGML_F16_DOT_VEC_REDUCE(sumf, sum);++ // leftovers+ for (int i = np; i < n; ++i) {+ sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));+ }+ // if you hit this, you are likely running outside the FP range+ assert(!isnan(sumf) && !isinf(sumf));+ #endif+#else+ for (int i = 0; i < n; ++i) {+ sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));+ }+#endif // GGML_SIMD++ *s = sumf;+}++void ggml_vec_silu_f32(const int n, float * y, const float * x) {+ int i = 0;+#if defined(__AVX512F__) && defined(__AVX512DQ__)+ for (; i + 15 < n; i += 16) {+ _mm512_storeu_ps(y + i, ggml_v_silu(_mm512_loadu_ps(x + i)));+ }+#elif defined(__AVX2__) && defined(__FMA__)+ for (; i + 7 < n; i += 8) {+ _mm256_storeu_ps(y + i, ggml_v_silu(_mm256_loadu_ps(x + i)));+ }+#elif defined(__SSE2__)+ for (; i + 3 < n; i += 4) {+ _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i)));+ }+#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)+ const int vlen = svcntw();+ for (; i < n; i += vlen) {+ const svbool_t pg = svwhilelt_b32_s32(i, n);+ svst1_f32(pg, y + i, ggml_v_silu(pg, svld1_f32(pg, x + i)));+ }+#elif defined(__ARM_NEON) && defined(__aarch64__)+ for (; i + 3 < n; i += 4) {+ vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));+ }+#elif defined(__riscv_v_intrinsic)+ for (int vl; i < n; i += vl) {+ vl = __riscv_vsetvl_e32m2(n - i);+ vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);+ vfloat32m2_t vy = ggml_v_silu_m2(vx, vl);+ __riscv_vse32_v_f32m2(&y[i], vy, vl);+ }+#endif+ for (; i < n; ++i) {+ y[i] = ggml_silu_f32(x[i]);+ }+}++void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g) {+ int i = 0;+#if defined(__AVX512F__) && defined(__AVX512DQ__)+ for (; i + 15 < n; i += 16) {+ _mm512_storeu_ps(y + i, _mm512_mul_ps(ggml_v_silu(_mm512_loadu_ps(x + i)), _mm512_loadu_ps(g + i)));+ }+#elif defined(__AVX2__) && defined(__FMA__)+ for (; i + 7 < n; i += 8) {+ _mm256_storeu_ps(y + i, _mm256_mul_ps(ggml_v_silu(_mm256_loadu_ps(x + i)), _mm256_loadu_ps(g + i)));+ }+#elif defined(__SSE2__)+ for (; i + 3 < n; i += 4) {+ _mm_storeu_ps(y + i, _mm_mul_ps(ggml_v_silu(_mm_loadu_ps(x + i)), _mm_loadu_ps(g + i)));+ }+#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)+ const int vlen = svcntw();+ for (; i < n; i += vlen) {+ const svbool_t pg = svwhilelt_b32_s32(i, n);+ svst1_f32(pg, y + i, svmul_f32_x(pg, ggml_v_silu(pg, svld1_f32(pg, x + i)), svld1_f32(pg, g + i)));+ }+#elif defined(__ARM_NEON) && defined(__aarch64__)+ for (; i + 3 < n; i += 4) {+ vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i)));+ }+#elif defined(__riscv_v_intrinsic)+ for (int vl; i < n; i += vl) {+ vl = __riscv_vsetvl_e32m2(n - i);+ vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);+ vfloat32m2_t vg = __riscv_vle32_v_f32m2(&g[i], vl);+ vfloat32m2_t vy = __riscv_vfmul_vv_f32m2(ggml_v_silu_m2(vx, vl), vg, vl);+ __riscv_vse32_v_f32m2(&y[i], vy, vl);+ }+#endif+ for (; i < n; ++i) {+ y[i] = ggml_silu_f32(x[i]) * g[i];+ }+}++ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean) {+ int i = 0;+ ggml_float sum = 0;+// TODO: optimize to process the remaining elements in groups using the smaller vector sizes from AVX2 and SSE+// ref: https://github.com/ggml-org/llama.cpp/pull/15953#pullrequestreview-3310928344+#if defined(__AVX512F__) && defined(__AVX512DQ__)+ for (; i + 15 < n; i += 16) {+ __m512 val = _mm512_sub_ps(_mm512_loadu_ps(x + i),+ _mm512_set1_ps(mean));+ _mm512_storeu_ps(y + i, val);+ sum += (ggml_float)_mm512_reduce_add_ps(_mm512_mul_ps(val, val));+ }+#elif defined(__AVX2__) && defined(__FMA__)+ for (; i + 7 < n; i += 8) {+ __m256 val = _mm256_sub_ps(_mm256_loadu_ps(x + i),+ _mm256_set1_ps(mean));+ _mm256_storeu_ps(y + i, val);+ val = _mm256_mul_ps(val,val);+ __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),+ _mm256_castps256_ps128(val));+ val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));+ val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));+ sum += (ggml_float)_mm_cvtss_f32(val2);+ }+#elif defined(__SSE2__)+ for (; i + 3 < n; i += 4) {+ __m128 val = _mm_sub_ps(_mm_loadu_ps(x + i),+ _mm_set1_ps(mean));+ _mm_storeu_ps(y + i, val);+ val = _mm_mul_ps(val, val);+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+ val = _mm_add_ps(val, _mm_movehl_ps(val, val));+ val = _mm_add_ss(val, _mm_movehdup_ps(val));+#else+ __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));+ val = _mm_add_ps(val, tmp);+ tmp = _mm_movehl_ps(tmp, val);+ val = _mm_add_ss(val, tmp);+#endif // __AVX__ || __AVX2__ || __AVX512F__+ sum += (ggml_float)_mm_cvtss_f32(val);+ }+#elif defined(__ARM_NEON) && defined(__aarch64__)+ for (; i + 3 < n; i += 4) {+ float32x4_t val = vsubq_f32(vld1q_f32(x + i),+ vdupq_n_f32(mean));+ vst1q_f32(y + i, val);+ val = vmulq_f32(val, val);+ sum += (ggml_float)vaddvq_f32(val);+ }+#elif defined(__VXE__) || defined(__VXE2__)+ for (; i + 3 < n; i += 4) {+ float32x4_t val = vec_sub(vec_xl(0, x + i), vec_splats(mean));+ vec_xst(val, 0, y + i);+ val = vec_mul(val, val);+ sum += (ggml_float)vec_hsum_f32x4(val);+ }+#elif defined(__riscv_v_intrinsic)+ vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);+ for (int vl; i < n; i += vl) {+ vl = __riscv_vsetvl_e32m2(n - i);+ vfloat32m2_t val = __riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], vl), mean, vl);+ __riscv_vse32_v_f32m2(&y[i], val, vl);+ val = __riscv_vfmul_vv_f32m2(val, val, vl);+ vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, vl);+ }+ sum = (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);+#endif+ for (; i < n; ++i) {+ float val = x[i] - mean;+ y[i] = val;+ val *= val;+ sum += (ggml_float)val;+ }+ return sum/n;+}++ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {+ int i = 0;+ ggml_float sum = 0;+#if defined(__AVX512F__) && defined(__AVX512DQ__)+ for (; i + 15 < n; i += 16) {+ __m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),+ _mm512_set1_ps(max)));+ _mm512_storeu_ps(y + i, val);+ sum += (ggml_float)_mm512_reduce_add_ps(val);+ }+#elif defined(__AVX2__) && defined(__FMA__)+ for (; i + 7 < n; i += 8) {+ __m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),+ _mm256_set1_ps(max)));+ _mm256_storeu_ps(y + i, val);+ __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),+ _mm256_castps256_ps128(val));+ val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));+ val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));+ sum += (ggml_float)_mm_cvtss_f32(val2);+ }+#elif defined(__SSE2__)+ for (; i + 3 < n; i += 4) {+ __m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),+ _mm_set1_ps(max)));+ _mm_storeu_ps(y + i, val);+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+ val = _mm_add_ps(val, _mm_movehl_ps(val, val));+ val = _mm_add_ss(val, _mm_movehdup_ps(val));+#else+ __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));+ val = _mm_add_ps(val, tmp);+ tmp = _mm_movehl_ps(tmp, val);+ val = _mm_add_ss(val, tmp);+#endif+ sum += (ggml_float)_mm_cvtss_f32(val);+ }+#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)+ const int vlen = svcntw();+ for (; i < n; i += vlen) {+ const svbool_t pg = svwhilelt_b32_s32(i, n);+ svfloat32_t val = ggml_v_expf(pg, svsub_f32_x(pg, svld1_f32(pg, x + i),+ svdup_n_f32_x(pg, max)));+ svst1_f32(pg, y + i, val);+ sum += (ggml_float)svaddv_f32(pg, val);+ }+#elif defined(__ARM_NEON) && defined(__aarch64__)+ for (; i + 3 < n; i += 4) {+ float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),+ vdupq_n_f32(max)));+ vst1q_f32(y + i, val);+ sum += (ggml_float)vaddvq_f32(val);+ }+#elif defined(__riscv_v_intrinsic)+ vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);+ for (int avl; i < n; i += avl) {+ avl = __riscv_vsetvl_e32m2(n - i);+ vfloat32m2_t val = ggml_v_expf_m2(__riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], avl), max, avl), avl);+ __riscv_vse32_v_f32m2(&y[i], val, avl);+ vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, avl);+ }+ return (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);+#endif+ for (; i < n; ++i) {+ float val = expf(x[i] - max);+ sum += (ggml_float)val;+ y[i] = val;+ }+ return sum;+}++ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max) {+ // log(soft_max) = log(soft_max_i / soft_max_sum) = log(soft_max_i) - log(soft_max_sum) = (logit_i - max) - log(soft_max_i)++ int i = 0;+ ggml_float sum = 0;+ for (; i < n; ++i) {+ float val = x[i] - max;+ y[i] = val;+ sum += (ggml_float)expf(val);+ }+ return sum = (ggml_float)logf(sum);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cpu/vec.h b/python/freetoken/kernel/csrc/ggml/ggml-cpu/vec.hnew file mode 100644index 00000000..59fa88f2--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cpu/vec.h@@ -0,0 +1,1630 @@+// Vectorized functions for fundamental operations++#pragma once++#include "ggml-impl.h"+#include "simd-mappings.h"+#include "ggml.h"+#include "ggml-cpu.h"++#if defined(GGML_USE_ACCELERATE)+#include <Accelerate/Accelerate.h>+#endif++// floating point type used to accumulate sums+typedef double ggml_float;++#if defined(__ARM_FEATURE_SVE)+inline static void ggml_sve_f16_fma_widened(+ svfloat32_t * acc_lo,+ svfloat32_t * acc_hi,+ svfloat16_t x,+ svfloat16_t y) {+#if defined(__ARM_FEATURE_SVE2)+ *acc_lo = svmlalb_f32(*acc_lo, x, y);+ *acc_hi = svmlalt_f32(*acc_hi, x, y);+#else+ // Plain SVE fallback path if SVE2 instructions not available+ svfloat16_t x_even = svtrn1_f16(x, x);+ svfloat16_t x_odd = svtrn2_f16(x, x);++ svfloat16_t y_even = svtrn1_f16(y, y);+ svfloat16_t y_odd = svtrn2_f16(y, y);++ svbool_t pg = svptrue_b32();++ *acc_lo = svmla_f32_x(pg, *acc_lo, svcvt_f32_f16_x(pg, x_even), svcvt_f32_f16_x(pg, y_even));+ *acc_hi = svmla_f32_x(pg, *acc_hi, svcvt_f32_f16_x(pg, x_odd), svcvt_f32_f16_x(pg, y_odd));+#endif+}++inline static ggml_float ggml_sve_sum_f32x2(svfloat32_t sum_lo, svfloat32_t sum_hi) {+ return (ggml_float) (svaddv_f32(svptrue_b32(), sum_lo) + svaddv_f32(svptrue_b32(), sum_hi));+}+#endif++#define GGML_GELU_FP16+#define GGML_GELU_QUICK_FP16++#define GGML_SOFT_MAX_UNROLL 4+#define GGML_VEC_DOT_UNROLL 2+#define GGML_VEC_MAD_UNROLL 32++#ifdef __cplusplus+extern "C" {+#endif++//+// global data+//++// precomputed gelu table for f16 (128 KB)+extern ggml_fp16_t ggml_table_gelu_f16[1 << 16];++// precomputed quick gelu table for f16 (128 KB)+extern ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];++//+// fundamental operations+//++void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);+void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc);+void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc);++void ggml_vec_silu_f32(const int n, float * y, const float * x);+ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean); //it will also center y ( y = y - mean )+ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);+ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max);++inline static void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }+inline static void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }++inline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }+inline static void ggml_vec_cpy_i32(const int n, int32_t * y, const int32_t * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }++inline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const ggml_fp16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }+inline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }++inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) {+ int i = 0;+#if defined(__AVX2__)+ for (; i + 7 < n; i += 8) {+ __m256 vx = _mm256_loadu_ps(x + i);+ __m256 vy = _mm256_loadu_ps(y + i);+ __m256 vz = _mm256_add_ps(vx, vy);+ _mm256_storeu_ps(z + i, vz);+ }+#endif+ for (; i < n; ++i) {+ z[i] = x[i] + y[i];+ }+}++inline static void ggml_vec_add_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {+ for (int i = 0; i < n; ++i) {+ z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) + GGML_CPU_FP16_TO_FP32(y[i]));+ }+}+inline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }+inline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }+inline static void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }+inline static void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }+inline static void ggml_vec_sub_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {+ for (int i = 0; i < n; ++i) {+ z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) - GGML_CPU_FP16_TO_FP32(y[i]));+ }+}+inline static void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }+inline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }+inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }+inline static void ggml_vec_neg_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(-GGML_CPU_FP16_TO_FP32(x[i]));+ }+}++inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }+inline static void ggml_vec_mul_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {+ for (int i = 0; i < n; ++i) {+ z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) * GGML_CPU_FP16_TO_FP32(y[i]));+ }+}+inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }+inline static void ggml_vec_div_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {+ for (int i = 0; i < n; ++i) {+ z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) / GGML_CPU_FP16_TO_FP32(y[i]));+ }+}++// compute GGML_VEC_DOT_UNROLL dot products at once+// xs - x row stride in bytes+inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GGML_RESTRICT s, void * GGML_RESTRICT xv, ggml_fp16_t * GGML_RESTRICT y) {+ ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };++ ggml_fp16_t * GGML_RESTRICT x[GGML_VEC_DOT_UNROLL];++ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {+ x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);+ }++#if defined(GGML_SIMD)+ #if defined(__ARM_FEATURE_SVE)++ const int ggml_f16_epr = svcnth();+ const int ggml_f16_step = 2 * ggml_f16_epr;+ int np = n - (n % ggml_f16_step);+ int np2 = n - (n % ggml_f16_epr);++ svfloat32_t sum_0_0_lo = svdup_n_f32(0.0f);+ svfloat32_t sum_0_0_hi = svdup_n_f32(0.0f);+ svfloat32_t sum_0_1_lo = svdup_n_f32(0.0f);+ svfloat32_t sum_0_1_hi = svdup_n_f32(0.0f);+ svfloat32_t sum_1_0_lo = svdup_n_f32(0.0f);+ svfloat32_t sum_1_0_hi = svdup_n_f32(0.0f);+ svfloat32_t sum_1_1_lo = svdup_n_f32(0.0f);+ svfloat32_t sum_1_1_hi = svdup_n_f32(0.0f);++ for (int i = 0; i < np; i += ggml_f16_step) {+ const svfloat16_t ay0 = GGML_F16x_VEC_LOAD(y + i, 0);+ const svfloat16_t ax00 = GGML_F16x_VEC_LOAD(x[0] + i, 0);+ const svfloat16_t ax01 = GGML_F16x_VEC_LOAD(x[1] + i, 0);++ ggml_sve_f16_fma_widened(&sum_0_0_lo, &sum_0_0_hi, ax00, ay0);+ ggml_sve_f16_fma_widened(&sum_1_0_lo, &sum_1_0_hi, ax01, ay0);++ const svfloat16_t ay1 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 0);+ const svfloat16_t ax10 = GGML_F16x_VEC_LOAD(x[0] + i + 1 * ggml_f16_epr, 0);+ const svfloat16_t ax11 = GGML_F16x_VEC_LOAD(x[1] + i + 1 * ggml_f16_epr, 0);++ ggml_sve_f16_fma_widened(&sum_0_1_lo, &sum_0_1_hi, ax10, ay1);+ ggml_sve_f16_fma_widened(&sum_1_1_lo, &sum_1_1_hi, ax11, ay1);+ }++ for (int i = np; i < np2; i += ggml_f16_epr) {+ const svfloat16_t ry = GGML_F16x_VEC_LOAD(y + i, 0);+ const svfloat16_t rx0 = GGML_F16x_VEC_LOAD(x[0] + i, 0);+ const svfloat16_t rx1 = GGML_F16x_VEC_LOAD(x[1] + i, 0);++ ggml_sve_f16_fma_widened(&sum_0_0_lo, &sum_0_0_hi, rx0, ry);+ ggml_sve_f16_fma_widened(&sum_1_0_lo, &sum_1_0_hi, rx1, ry);+ }++ if (np2 < n) {+ const svbool_t pg = svwhilelt_b16(np2, n);+ const svfloat16_t ay = svld1_f16(pg, (const __fp16 *)(y + np2));+ const svfloat16_t ax0 = svld1_f16(pg, (const __fp16 *)(x[0] + np2));+ const svfloat16_t ax1 = svld1_f16(pg, (const __fp16 *)(x[1] + np2));++ ggml_sve_f16_fma_widened(&sum_0_0_lo, &sum_0_0_hi, ax0, ay);+ ggml_sve_f16_fma_widened(&sum_1_0_lo, &sum_1_0_hi, ax1, ay);+ }++ svfloat32_t sum_0_lo = svadd_f32_x(DEFAULT_PG32, sum_0_0_lo, sum_0_1_lo);+ svfloat32_t sum_0_hi = svadd_f32_x(DEFAULT_PG32, sum_0_0_hi, sum_0_1_hi);+ svfloat32_t sum_1_lo = svadd_f32_x(DEFAULT_PG32, sum_1_0_lo, sum_1_1_lo);+ svfloat32_t sum_1_hi = svadd_f32_x(DEFAULT_PG32, sum_1_0_hi, sum_1_1_hi);+ sumf[0] = ggml_sve_sum_f32x2(sum_0_lo, sum_0_hi);+ sumf[1] = ggml_sve_sum_f32x2(sum_1_lo, sum_1_hi);+ np = n;+ #elif defined(__riscv_v_intrinsic)+ #if defined(__riscv_zvfh)+ size_t vl = __riscv_vsetvlmax_e32m4();++ // initialize accumulators to all zeroes+ vfloat32m4_t vsum0_0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);+ vfloat32m4_t vsum0_1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);+ vfloat32m4_t vsum1_0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);+ vfloat32m4_t vsum1_1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);++ // calculate step size+ const size_t epr = __riscv_vsetvlmax_e16m2();+ const size_t step = epr * 2;+ int np = (n & ~(step - 1));++ // unroll by 2 along the row dimension+ for (int i = 0; i < np; i += step) {+ vfloat16m2_t ay0 = __riscv_vle16_v_f16m2((const _Float16 *)(y + i), epr);+ vfloat16m2_t ax0_0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i), epr);+ vfloat16m2_t ax1_0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i), epr);+ vsum0_0 = __riscv_vfwmacc_vv_f32m4(vsum0_0, ax0_0, ay0, epr);+ vsum1_0 = __riscv_vfwmacc_vv_f32m4(vsum1_0, ax1_0, ay0, epr);++ vfloat16m2_t ay1 = __riscv_vle16_v_f16m2((const _Float16 *)(y + i + epr), epr);+ vfloat16m2_t ax0_1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i + epr), epr);+ vfloat16m2_t ax1_1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i + epr), epr);+ vsum0_1 = __riscv_vfwmacc_vv_f32m4(vsum0_1, ax0_1, ay1, epr);+ vsum1_1 = __riscv_vfwmacc_vv_f32m4(vsum1_1, ax1_1, ay1, epr);+ }++ vfloat32m4_t vsum0 = __riscv_vfadd_vv_f32m4(vsum0_0, vsum0_1, vl);+ vfloat32m4_t vsum1 = __riscv_vfadd_vv_f32m4(vsum1_0, vsum1_1, vl);++ // leftovers+ for (int i = np; i < n; i += vl) {+ vl = __riscv_vsetvl_e16m2(n - i);+ vfloat16m2_t ay = __riscv_vle16_v_f16m2((const _Float16 *)(y + i), vl);+ vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i), vl);+ vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i), vl);++ vsum0 = __riscv_vfwmacc_vv_f32m4(vsum0, ax0, ay, vl);+ vsum1 = __riscv_vfwmacc_vv_f32m4(vsum1, ax1, ay, vl);+ }++ // reduce+ vl = __riscv_vsetvlmax_e32m2();+ vfloat32m2_t acc0_0 = __riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(vsum0, 0),+ __riscv_vget_v_f32m4_f32m2(vsum0, 1), vl);+ vl = __riscv_vsetvlmax_e32m1();+ vfloat32m1_t acc0_1 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(acc0_0, 0),+ __riscv_vget_v_f32m2_f32m1(acc0_0, 1), vl);+ vfloat32m1_t redsum0 = __riscv_vfredusum_vs_f32m1_f32m1(+ acc0_1, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);++ vl = __riscv_vsetvlmax_e32m2();+ vfloat32m2_t acc1_0 = __riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(vsum1, 0),+ __riscv_vget_v_f32m4_f32m2(vsum1, 1), vl);+ vl = __riscv_vsetvlmax_e32m1();+ vfloat32m1_t acc1_1 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(acc1_0, 0),+ __riscv_vget_v_f32m2_f32m1(acc1_0, 1), vl);+ vfloat32m1_t redsum1 = __riscv_vfredusum_vs_f32m1_f32m1(+ acc1_1, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);+ sumf[0] = __riscv_vfmv_f_s_f32m1_f32(redsum0);+ sumf[1] = __riscv_vfmv_f_s_f32m1_f32(redsum1);+ np = n;+ #else+ const int np = 0;+ #endif+ #else+ const int np = (n & ~(GGML_F16_DOT_STEP - 1));++ GGML_F16_DOT_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_DOT_ARR] = { { GGML_F16_DOT_VEC_ZERO } };++ GGML_F16_DOT_VEC ax[GGML_F16_DOT_ARR];+ GGML_F16_DOT_VEC ay[GGML_F16_DOT_ARR];++ for (int i = 0; i < np; i += GGML_F16_DOT_STEP) {+ for (int j = 0; j < GGML_F16_DOT_ARR; j++) {+ ay[j] = GGML_F16_DOT_VEC_LOAD(y + i + j*GGML_F16_DOT_EPR, j);++ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {+ ax[j] = GGML_F16_DOT_VEC_LOAD(x[k] + i + j*GGML_F16_DOT_EPR, j);++ sum[k][j] = GGML_F16_DOT_VEC_FMA(sum[k][j], ax[j], ay[j]);+ }+ }+ }++ // reduce sum0..sum3 to sum0+ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {+ GGML_F16_DOT_VEC_REDUCE(sumf[k], sum[k]);+ }+ #endif+#else+ // scalar path+ const int np = 0;+#endif+ // scalar and leftovers+ for (int i = np; i < n; ++i) {+ for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {+ sumf[j] += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[j][i])*GGML_CPU_FP16_TO_FP32(y[i]));+ }+ }++ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {+ s[i] = (float)sumf[i];+ }+}++inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const float * GGML_RESTRICT x, const float v) {+#if defined(GGML_SIMD)+ #if defined(__ARM_FEATURE_SVE)++ const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;+ const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16+ const int ggml_f32_step = 8 * ggml_f32_epr; // choose 8 SVE registers+ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);++ const int np = (n & ~(ggml_f32_step - 1));+ svfloat32_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;+ svfloat32_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;+ for (int i = 0; i < np; i += ggml_f32_step) {++ ax1 = GGML_F32_VEC_LOAD(x + i);+ ay1 = GGML_F32_VEC_LOAD(y + i);+ ay1 = GGML_F32_VEC_FMA(ay1, ax1, vx);++ GGML_F32_VEC_STORE(y + i, ay1);++ ax2 = GGML_F32_VEC_LOAD(x + i + 1*ggml_f32_epr);+ ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);+ ay2 = GGML_F32_VEC_FMA(ay2, ax2, vx);++ GGML_F32_VEC_STORE(y + i + 1*ggml_f32_epr, ay2);++ ax3 = GGML_F32_VEC_LOAD(x + i + 2*ggml_f32_epr);+ ay3 = GGML_F32_VEC_LOAD(y + i + 2*ggml_f32_epr);+ ay3 = GGML_F32_VEC_FMA(ay3, ax3, vx);++ GGML_F32_VEC_STORE(y + i + 2*ggml_f32_epr, ay3);++ ax4 = GGML_F32_VEC_LOAD(x + i + 3*ggml_f32_epr);+ ay4 = GGML_F32_VEC_LOAD(y + i + 3*ggml_f32_epr);+ ay4 = GGML_F32_VEC_FMA(ay4, ax4, vx);++ GGML_F32_VEC_STORE(y + i + 3*ggml_f32_epr, ay4);++ ax5 = GGML_F32_VEC_LOAD(x + i + 4*ggml_f32_epr);+ ay5 = GGML_F32_VEC_LOAD(y + i + 4*ggml_f32_epr);+ ay5 = GGML_F32_VEC_FMA(ay5, ax5, vx);++ GGML_F32_VEC_STORE(y + i + 4*ggml_f32_epr, ay5);++ ax6 = GGML_F32_VEC_LOAD(x + i + 5*ggml_f32_epr);+ ay6 = GGML_F32_VEC_LOAD(y + i + 5*ggml_f32_epr);+ ay6 = GGML_F32_VEC_FMA(ay6, ax6, vx);++ GGML_F32_VEC_STORE(y + i + 5*ggml_f32_epr, ay6);++ ax7 = GGML_F32_VEC_LOAD(x + i + 6*ggml_f32_epr);+ ay7 = GGML_F32_VEC_LOAD(y + i + 6*ggml_f32_epr);+ ay7 = GGML_F32_VEC_FMA(ay7, ax7, vx);++ GGML_F32_VEC_STORE(y + i + 6*ggml_f32_epr, ay7);++ ax8 = GGML_F32_VEC_LOAD(x + i + 7*ggml_f32_epr);+ ay8 = GGML_F32_VEC_LOAD(y + i + 7*ggml_f32_epr);+ ay8 = GGML_F32_VEC_FMA(ay8, ax8, vx);++ GGML_F32_VEC_STORE(y + i + 7*ggml_f32_epr, ay8);+ }+ // leftovers+ // Since 8 unrolls are done in above loop, leftovers lie in range [0, ggml_f32_step] which is handled in below loop+ const int np2 = (n & ~(ggml_f32_epr - 1));+ for (int i = np; i < np2; i += ggml_f32_epr) {+ ax1 = GGML_F32_VEC_LOAD(x + i);+ ay1 = GGML_F32_VEC_LOAD(y + i);+ ay1 = GGML_F32_VEC_FMA(ay1, ax1, vx);++ GGML_F32_VEC_STORE(y + i, ay1);+ }+ // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only+ if (np2 < n) {+ svbool_t pg =svwhilelt_b32(np2, n);+ ax1 = svld1_f32(pg, x + np2);+ ay1 = svld1_f32(pg, y + np2);+ ay1 = svmad_f32_m(pg, ax1, vx, ay1);++ svst1_f32(pg, y + np2, ay1);+ }+ #elif defined(__riscv_v_intrinsic)+ for (int i = 0, avl; i < n; i += avl) {+ avl = __riscv_vsetvl_e32m8(n - i);+ vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);+ vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);+ vfloat32m8_t ny = __riscv_vfmadd_vf_f32m8(ax, v, ay, avl);+ __riscv_vse32_v_f32m8(&y[i], ny, avl);+ }+ #else+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);++ GGML_F32_VEC ax[GGML_F32_ARR];+ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);++ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] += x[i]*v;+ }+ #endif+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] += x[i]*v;+ }+#endif+}++inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, const ggml_fp16_t * GGML_RESTRICT x, const float v) {+#if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE)+ const int sve_register_length = svcntb() * 8;+ const int ggml_f16_epr = sve_register_length / 16;+ const int ggml_f16_step = 8 * ggml_f16_epr;++ GGML_F16x_VEC vx = GGML_F16x_VEC_SET1(v);++ int np = (n & ~(ggml_f16_step - 1));++ svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;+ svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;+ for (int i = 0; i < np; i += ggml_f16_step) {+ ax1 = GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0);+ ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0);+ ay1 = GGML_F16x_VEC_FMA(ay1, ax1, vx);++ GGML_F16x_VEC_STORE(y + i + 0 * ggml_f16_epr, ay1, 0);++ ax2 = GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1);+ ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1);+ ay2 = GGML_F16x_VEC_FMA(ay2, ax2, vx);++ GGML_F16x_VEC_STORE(y + i + 1 * ggml_f16_epr, ay2, 1);++ ax3 = GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2);+ ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2);+ ay3 = GGML_F16x_VEC_FMA(ay3, ax3, vx);++ GGML_F16x_VEC_STORE(y + i + 2 * ggml_f16_epr, ay3, 2);++ ax4 = GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3);+ ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3);+ ay4 = GGML_F16x_VEC_FMA(ay4, ax4, vx);++ GGML_F16x_VEC_STORE(y + i + 3 * ggml_f16_epr, ay4, 3);++ ax5 = GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4);+ ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4);+ ay5 = GGML_F16x_VEC_FMA(ay5, ax5, vx);++ GGML_F16x_VEC_STORE(y + i + 4 * ggml_f16_epr, ay5, 4);++ ax6 = GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5);+ ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5);+ ay6 = GGML_F16x_VEC_FMA(ay6, ax6, vx);++ GGML_F16x_VEC_STORE(y + i + 5 * ggml_f16_epr, ay6, 5);++ ax7 = GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6);+ ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6);+ ay7 = GGML_F16x_VEC_FMA(ay7, ax7, vx);++ GGML_F16x_VEC_STORE(y + i + 6 * ggml_f16_epr, ay7, 6);++ ax8 = GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7);+ ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7);+ ay8 = GGML_F16x_VEC_FMA(ay8, ax8, vx);++ GGML_F16x_VEC_STORE(y + i + 7 * ggml_f16_epr, ay8, 7);+ }+ const int np2 = (n & ~(ggml_f16_epr - 1));+ for (int k = np; k < np2; k += ggml_f16_epr) {+ svfloat16_t rx = GGML_F16x_VEC_LOAD(x + k, 0);+ svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0);+ ry = GGML_F16x_VEC_FMA(ry, rx, vx);++ GGML_F16x_VEC_STORE(y + k, ry, 0);+ }++ if (np2 < n) {+ svbool_t pg = svwhilelt_b16(np2, n);+ svfloat16_t hx = svld1_f16(pg, (const __fp16 *)(x + np2));+ svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2));+ hy = svmad_f16_x(pg, hx, vx, hy);+ svst1_f16(pg, (__fp16 *)(y + np2), hy);+ }+ np = n;+#elif defined(__riscv_v_intrinsic) // implies __riscv_v_intrinsic+ #if defined (__riscv_zvfh)+ const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);+ const _Float16 scale = *(const _Float16*)(&s);++ // calculate step size+ const int epr = __riscv_vsetvlmax_e16m4();+ const int step = epr * 2;+ int np = (n & ~(step - 1));++ // unroll by 2+ for (int i = 0; i < np; i += step) {+ vfloat16m4_t ax0 = __riscv_vle16_v_f16m4((const _Float16*)x + i, epr);+ vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);+ ay0 = __riscv_vfmacc_vf_f16m4(ay0, scale, ax0, epr);+ __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);+ __asm__ __volatile__ ("" ::: "memory");++ vfloat16m4_t ax1 = __riscv_vle16_v_f16m4((const _Float16*)x + i + epr, epr);+ vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);+ ay1 = __riscv_vfmacc_vf_f16m4(ay1, scale, ax1, epr);+ __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);+ __asm__ __volatile__ ("" ::: "memory");+ }++ // leftovers+ int vl;+ for (int i = np; i < n; i += vl) {+ vl = __riscv_vsetvl_e16m4(n - i);+ vfloat16m4_t ax0 = __riscv_vle16_v_f16m4((const _Float16*)x + i, vl);+ vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, vl);+ ay0 = __riscv_vfmacc_vf_f16m4(ay0, scale, ax0, vl);+ __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, vl);+ }+ np = n;+ #else+ // fall to scalar path+ const int np = 0;+ #endif+#elif defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);++ GGML_F16_VEC ax[GGML_F16_ARR];+ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);++ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);+ }+ }+#else+ // scalar path+ const int np = 0;+#endif++ // scalar and leftovers+ for (int i = np; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i]) + GGML_CPU_FP16_TO_FP32(x[i])*v);+ }+}++// xs and vs are byte strides of x and v+inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * GGML_RESTRICT y, const float * GGML_RESTRICT xv, const float * GGML_RESTRICT vv) {++ const float * GGML_RESTRICT x[GGML_VEC_MAD_UNROLL];+ const float * GGML_RESTRICT v[GGML_VEC_MAD_UNROLL];++ for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {+ x[i] = (const float *) ((const char *) xv + i*xs);+ v[i] = (const float *) ((const char *) vv + i*vs);+ }++#if defined(GGML_SIMD)+ #if defined(__ARM_FEATURE_SVE)+ // scalar Route to scalar implementation //TODO: Write SVE code+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ for (int i = 0; i < n; ++i) {+ y[i] += x[k][i]*v[k][0];+ }+ }+ #elif defined(__riscv_v_intrinsic)+ for (int i = 0, avl; i < n; i += avl) {+ avl = __riscv_vsetvl_e32m8(n - i);+ vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; k++) {+ vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[k][i], avl);+ ay = __riscv_vfmadd_vf_f32m8(ax, v[k][0], ay, avl);+ }+ __riscv_vse32_v_f32m8(&y[i], ay, avl);+ }+ #else+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];++ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ vx[k] = GGML_F32_VEC_SET1(v[k][0]);+ }++ GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];+ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);++ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);+ }++ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);+ }+ }++ // leftovers+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ for (int i = np; i < n; ++i) {+ y[i] += x[k][i]*v[k][0];+ }+ }+ #endif+#else+ // scalar+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ for (int i = 0; i < n; ++i) {+ y[i] += x[k][i]*v[k][0];+ }+ }+#endif+}++inline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, const float s, const float b) {+#if defined(GGML_USE_ACCELERATE)+ vDSP_vsmsa(x, 1, &s, &b, y, 1, n);+#elif defined(GGML_SIMD)+ #if defined(__ARM_FEATURE_SVE)+ // scalar ; TODO: Write SVE code+ for (int i = 0; i < n; ++i) {+ y[i] = x[i]*s + b;+ }+ #elif defined(__riscv_v_intrinsic)+ for (int i = 0, avl; i < n; i += avl) {+ avl = __riscv_vsetvl_e32m8(n - i);+ vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);+ vfloat32m8_t vb = __riscv_vfmv_v_f_f32m8(b, avl);+ vfloat32m8_t ny = __riscv_vfmadd_vf_f32m8(ax, s, vb, avl);+ __riscv_vse32_v_f32m8(&y[i], ny, avl);+ }+ #else+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC vs = GGML_F32_VEC_SET1(s);+ GGML_F32_VEC vb = GGML_F32_VEC_SET1(b);++ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ay[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_FMA(vb, ay[j], vs);++ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] = x[i]*s + b;+ }+ #endif+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] = x[i]*s + b;+ }+#endif+}++//inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }+inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {+#if defined(GGML_USE_ACCELERATE)+ vDSP_vsmul(y, 1, &v, y, 1, n);+#elif defined(GGML_SIMD)+ #if defined(__ARM_FEATURE_SVE)+ const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;+ const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16+ const int ggml_f32_step = 2 * ggml_f32_epr;++ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);+ const int np = (n & ~(ggml_f32_step - 1));+ svfloat32_t ay1;+ svfloat32_t ay2;+ for (int i = 0; i < np; i += ggml_f32_step) {+ ay1 = GGML_F32_VEC_LOAD(y + i);+ ay1 = GGML_F32_VEC_MUL(ay1, vx);+ GGML_F32_VEC_STORE(y + i, ay1);++ ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);+ ay2 = GGML_F32_VEC_MUL(ay2, vx);+ GGML_F32_VEC_STORE(y + i + 1*ggml_f32_epr, ay2);+ }+ // leftovers+ // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only+ for (int i = np; i < n; i += ggml_f32_epr) {+ svbool_t pg = svwhilelt_b32(i, n);+ ay1 = svld1_f32(pg, y + i);+ ay1 = svmul_f32_m(pg, ay1, vx);+ svst1_f32(pg, y + i, ay1);+ }+ #elif defined(__riscv_v_intrinsic)+ for (int i = 0, avl; i < n; i += avl) {+ avl = __riscv_vsetvl_e32m8(n - i);+ vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);+ vfloat32m8_t ny = __riscv_vfmul_vf_f32m8(ay, v, avl);+ __riscv_vse32_v_f32m8(&y[i], ny, avl);+ }+ #else+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);++ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_MUL(ay[j], vx);++ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] *= v;+ }+ #endif+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] *= v;+ }+#endif+}++inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {+#if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE)+ const int sve_register_length = svcntb() * 8;+ const int ggml_f16_epr = sve_register_length / 16;+ const int ggml_f16_step = 2 * ggml_f16_epr;++ GGML_F16x_VEC vx = GGML_F16x_VEC_SET1(v);+ int np = (n & ~(ggml_f16_step - 1));+ svfloat16_t ay1, ay2;++ for (int i = 0; i < np; i += ggml_f16_step) {+ ay1 = GGML_F16x_VEC_LOAD(y + i + 0*ggml_f16_epr, 0);+ ay1 = GGML_F16x_VEC_MUL(ay1, vx);+ GGML_F16x_VEC_STORE(y + i + 0*ggml_f16_epr, ay1, 0);++ ay2 = GGML_F16x_VEC_LOAD(y + i + 1*ggml_f16_epr, 1);+ ay2 = GGML_F16x_VEC_MUL(ay2, vx);+ GGML_F16x_VEC_STORE(y + i + 1*ggml_f16_epr, ay2, 1);+ }+ // leftovers+ // maximum number of leftover elements will be less that ggmlF_16x_epr. Apply predicated svmad on available elements only+ if (np < n) {+ svbool_t pg = svwhilelt_b16(np, n);+ svfloat16_t hy = svld1_f16(pg, (__fp16 *)(y + np));+ svfloat16_t out = svmul_f16_m(pg, hy, vx);+ svst1_f16(pg, (__fp16 *)(y + np), out);+ }+ np = n;+#elif defined(__riscv_v_intrinsic)+ #if defined(__riscv_zvfh)+ const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);+ const _Float16 scale = *(const _Float16*)(&s);++ // calculate step size+ const int epr = __riscv_vsetvlmax_e16m4();+ const int step = epr * 2;+ int np = (n & ~(step - 1));++ // unroll by 2+ for (int i = 0; i < np; i += step) {+ vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);+ ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, epr);+ __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);+ __asm__ __volatile__ ("" ::: "memory");++ vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);+ ay1 = __riscv_vfmul_vf_f16m4(ay1, scale, epr);+ __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);+ __asm__ __volatile__ ("" ::: "memory");+ }++ // leftovers+ int vl;+ for (int i = np; i < n; i += vl) {+ vl = __riscv_vsetvl_e16m4(n - i);+ vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, vl);+ ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, vl);+ __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, vl);+ }+ np = n;+ #else+ // fall to scalar path+ const int np = 0;+ #endif+#elif defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);++ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_MUL(ay[j], vx);++ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);+ }+ }+#else+ // scalar path+ const int np = 0;+#endif+ // scalar and leftovers+ for (int i = np; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i])*v);+ }+}++inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }+inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }+inline static void ggml_vec_sqr_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(x[i]);+ y[i] = GGML_CPU_FP32_TO_FP16(v*v);+ }+}+inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }+inline static void ggml_vec_sqrt_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(sqrtf(GGML_CPU_FP16_TO_FP32(x[i])));+ }+}+inline static void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }+inline static void ggml_vec_log_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(logf(GGML_CPU_FP16_TO_FP32(x[i])));+ }+}+inline static void ggml_vec_sin_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sinf(x[i]); }+inline static void ggml_vec_sin_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(sinf(GGML_CPU_FP16_TO_FP32(x[i])));+ }+}+inline static void ggml_vec_cos_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = cosf(x[i]); }+inline static void ggml_vec_cos_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(cosf(GGML_CPU_FP16_TO_FP32(x[i])));+ }+}+inline static void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }+inline static void ggml_vec_abs_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(fabsf(GGML_CPU_FP16_TO_FP32(x[i])));+ }+}+inline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }+inline static void ggml_vec_sgn_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(x[i]);+ y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? 1.f : ((v < 0.f) ? -1.f : 0.f));+ }+}+inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }+inline static void ggml_vec_step_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16((GGML_CPU_FP16_TO_FP32(x[i]) > 0.f) ? 1.f : 0.f);+ }+}+inline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }+inline static void ggml_vec_tanh_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(tanhf(GGML_CPU_FP16_TO_FP32(x[i])));+ }+}+inline static void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expm1f(x[i]); }+inline static void ggml_vec_elu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ const float v = GGML_CPU_FP16_TO_FP32(x[i]);+ y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v : expm1f(v));+ }+}+inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }+inline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(x[i]);+ y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v : 0.f);+ }+}+inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }+inline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) {+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(x[i]);+ y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f));+ }+}+inline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }+inline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(1.f / (1.f + expf(-GGML_CPU_FP16_TO_FP32(x[i]))));+ }+}+// TODO: optimize performance+inline static void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }+inline static void ggml_vec_hardswish_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(x[i]);+ y[i] = GGML_CPU_FP32_TO_FP16(v * fminf(1.0f, fmaxf(0.0f, (v + 3.0f) / 6.0f)));+ }+}+inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }+inline static void ggml_vec_hardsigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(fminf(1.0f, fmaxf(0.0f, (GGML_CPU_FP16_TO_FP32(x[i]) + 3.0f) / 6.0f)));+ }+}+inline static void ggml_vec_exp_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = expf(x[i]); }+inline static void ggml_vec_exp_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_CPU_FP32_TO_FP16(expf(GGML_CPU_FP16_TO_FP32(x[i])));+ }+}++static const float GELU_COEF_A = 0.044715f;+static const float GELU_QUICK_COEF = -1.702f;+static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;+static const float SQRT_2_INV = 0.70710678118654752440084436210484f;++inline static float ggml_gelu_f32(float x) {+ return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));+}++inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ const uint16_t * i16 = (const uint16_t *) x;+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_table_gelu_f16[i16[i]];+ }+}++inline static void ggml_vec_gelu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_BF16(ggml_gelu_f32(GGML_BF16_TO_FP32(x[i])));+ }+}++inline static void ggml_vec_gelu_erf_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ float xi = GGML_CPU_FP16_TO_FP32(x[i]);+ float res = 0.5f*xi*(1.0f + erff(xi*SQRT_2_INV));+ y[i] = GGML_CPU_FP32_TO_FP16(res);+ }+}++inline static void ggml_vec_gelu_erf_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) {+ for (int i = 0; i < n; ++i) {+ float xi = GGML_BF16_TO_FP32(x[i]);+ y[i] = GGML_FP32_TO_BF16(0.5f*xi*(1.0f + erff(xi*SQRT_2_INV)));+ }+}++#ifdef GGML_GELU_FP16+inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {+ uint16_t t;+ for (int i = 0; i < n; ++i) {+ if (x[i] <= -10.0f) {+ y[i] = 0.0f;+ } else if (x[i] >= 10.0f) {+ y[i] = x[i];+ } else {+ ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);+ memcpy(&t, &fp16, sizeof(uint16_t));+ y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[t]);+ }+ }+}+#else+inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_gelu_f32(x[i]);+ }+}+#endif++inline static void ggml_vec_gelu_erf_f32(const int n, float * y, const float * x) {+ for (int i = 0; i < n; ++i) {+ float xi = x[i];+ y[i] = 0.5f*xi*(1.0f + erff(xi*SQRT_2_INV));+ }+}++inline static float ggml_gelu_quick_f32(float x) {+ return x*(1.0f/(1.0f+expf(GELU_QUICK_COEF*x)));+}++inline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ const uint16_t * i16 = (const uint16_t *) x;+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_table_gelu_quick_f16[i16[i]];+ }+}++inline static void ggml_vec_gelu_quick_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_BF16(ggml_gelu_quick_f32(GGML_BF16_TO_FP32(x[i])));+ }+}++#ifdef GGML_GELU_QUICK_FP16+inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {+ uint16_t t;+ for (int i = 0; i < n; ++i) {+ ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);+ memcpy(&t, &fp16, sizeof(uint16_t));+ y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[t]);+ }+}+#else+inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_gelu_quick_f32(x[i]);+ }+}+#endif++// Sigmoid Linear Unit (SiLU) function+inline static float ggml_silu_f32(float x) {+ return x/(1.0f + expf(-x));+}+inline static ggml_fp16_t ggml_silu_f16(ggml_fp16_t x) {+ float v = GGML_CPU_FP16_TO_FP32(x);+ return GGML_CPU_FP32_TO_FP16(v/(1.0f + expf(-v)));+}++#if __FINITE_MATH_ONLY__+#error "some routines in ggml.c require non-finite math arithmetics -- pass -fno-finite-math-only to the compiler to fix"+#error "ref: https://github.com/ggml-org/llama.cpp/pull/7154#issuecomment-2143844461"+#endif++/* Below function was borrowed from the GitHub repository:+https://github.com/openvinotoolkit/openvino/blob/master/src/plugins/intel_cpu/src/nodes/kernels/scaled_attn/common.hpp */+#if defined(__ARM_FEATURE_SVE) && defined(__aarch64__)+ inline static svfloat32_t exp_ps_sve(svbool_t pg, svfloat32_t src) {+ // Constants+ const svfloat32_t log2_e = svdup_n_f32(1.4426950409f);+ const svfloat32_t ln2 = svdup_n_f32(0.6931473921f);+ const svfloat32_t half_ln2_sq = svdup_n_f32(0.2413862043f);+ const svuint32_t not_mask17 = svdup_n_u32(~((1u << 17) - 1));+ const svfloat32_t one = svdup_n_f32(1.0f);+ const svfloat32_t inactive1 = svdup_n_f32(0.0f);+ const svint32_t inactive2 = svdup_n_s32(0);++ // Algorithm starts here+ svfloat32_t t0 = svmul_f32_m(pg, src, log2_e); // y = x * log2(e)+ svfloat32_t t1 = svrintm_f32_m(inactive1, pg, t0); // rount to int (float)+ svint32_t t2 = svcvt_s32_f32_m(inactive2, pg, t1); // n++ t1 = svsub_f32_m(pg, t0, t1); // a = y - floor(y)+ t1 = svadd_f32_m(pg, t1, one); // b = a + 1++ svuint32_t t3 = svlsr_n_u32_m(pg, svreinterpret_u32_f32(t1), 17); // v = b >> 17 (u32)+ svfloat32_t t4 = svexpa_f32(t3); // c = fexpa(v)+ t4 = svscale_f32_m(pg, t4, t2); // fexpa(v) * 2^(n)++ // and_(t2.d, t1.d, not_mask17.d)+ svfloat32_t t5 = svreinterpret_f32_u32(svand_u32_m(pg, svreinterpret_u32_f32(t1), not_mask17));+ t5 = svsub_f32_m(pg, t1, t5); // z+ t0 = svmla_f32_m(pg, ln2, t5, half_ln2_sq); // ln2 + half_ln2_sq * z+ t0 = svmla_f32_m(pg, one, t5, t0); // 1 + (ln2 * z) + (half_ln2_sq * z * z)+ t0 = svmul_f32_m(pg, t0, t4); // Final result++ return t0;+ }+#endif++#if defined(__ARM_FEATURE_SVE) && defined(__aarch64__)++inline static svfloat32_t ggml_v_expf(svbool_t pg, svfloat32_t x) {+ const svfloat32_t r = svdup_n_f32_x(pg, 0x1.8p23f);+ const svfloat32_t z = svmla_n_f32_x(pg, r, x, 0x1.715476p+0f);+ const svfloat32_t n = svsub_f32_x(pg, z, r);+ const svfloat32_t b = svmls_n_f32_x(pg, svmls_n_f32_x(pg, x, n, 0x1.62e4p-1f), n, 0x1.7f7d1cp-20f);+ const svuint32_t e = svlsl_n_u32_x(pg, svreinterpret_u32_f32(z), 23);+ const svfloat32_t k = svreinterpret_f32_u32(svadd_u32_x(pg, e, svreinterpret_u32_f32(svdup_n_f32_x(pg, 1))));+ const svbool_t c = svacgt_n_f32(pg, n, 126);+ const svfloat32_t u = svmul_f32_x(pg, b, b);+ const svfloat32_t j = svmla_f32_x(pg,+ svmul_n_f32_x(pg, b, 0x1.ffffecp-1f),+ svmla_f32_x(pg, svmla_f32_x(pg, svdup_n_f32_x(pg, 0x1.fffdb6p-2f), svdup_n_f32_x(pg, 0x1.555e66p-3f), b),+ svmla_f32_x(pg, svdup_n_f32_x(pg, 0x1.573e2ep-5f), svdup_n_f32_x(pg, 0x1.0e4020p-7f), b), u), u);+ const svuint32_t d = svdup_n_u32_z(svcmple_n_f32(pg, n, 0.0), 0x82000000);+ const svfloat32_t s1 = svreinterpret_f32_u32(svadd_n_u32_x(pg, d, 0x7f000000));+ const svfloat32_t s2 = svreinterpret_f32_u32(svsub_u32_x(pg, e, d));+ return svsel_f32(svacgt_f32(pg, n, svdup_n_f32_x(pg, 192)), svmul_f32_x(pg, s1, s1),+ svsel_f32(c, svmul_f32_x(pg, svmla_f32_x(pg, s2, s2, j), s1), svmla_f32_x(pg, k, k, j)));+}++// computes silu x/(1+exp(-x)) in single precision vector+inline static svfloat32_t ggml_v_silu(svbool_t pg, svfloat32_t x) {+ const svfloat32_t one = svdup_n_f32_x(pg, 1.0f);+ const svfloat32_t zero = svdup_n_f32_x(pg, 0.0f);+ const svfloat32_t neg_x = svsub_f32_x(pg, zero, x);+ const svfloat32_t exp_neg_x = ggml_v_expf(pg, neg_x);+ const svfloat32_t one_plus_exp_neg_x = svadd_f32_x(pg, one, exp_neg_x);+ return svdiv_f32_x(pg, x, one_plus_exp_neg_x);+}++#elif defined(__ARM_NEON) && defined(__aarch64__)++// adapted from arm limited optimized routine+// the maximum error is 1.45358 plus 0.5 ulps+// numbers above 88.38 will flush to infinity+// numbers beneath -103.97 will flush to zero+inline static float32x4_t ggml_v_expf(float32x4_t x) {+ const float32x4_t r = vdupq_n_f32(0x1.8p23f);+ const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f));+ const float32x4_t n = vsubq_f32(z, r);+ const float32x4_t b = vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n,+ vdupq_n_f32(0x1.7f7d1cp-20f));+ const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23);+ const float32x4_t k = vreinterpretq_f32_u32(vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1))));+ const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126));+ const float32x4_t u = vmulq_f32(b, b);+ const float32x4_t j = vfmaq_f32(+ vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b),+ vfmaq_f32(vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), vdupq_n_f32(0x1.555e66p-3f), b),+ vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), vdupq_n_f32(0x1.0e4020p-7f), b), u), u);+ if (!vpaddd_u64(vreinterpretq_u64_u32(c)))+ return vfmaq_f32(k, j, k);+ const uint32x4_t d = vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000));+ const float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000)));+ const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d));+ return vbslq_f32(vcagtq_f32(n, vdupq_n_f32(192)), vmulq_f32(s1, s1),+ vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), vfmaq_f32(k, k, j)));+}++// computes silu x/(1+exp(-x)) in single precision vector+inline static float32x4_t ggml_v_silu(float32x4_t x) {+ const float32x4_t one = vdupq_n_f32(1.0f);+ const float32x4_t zero = vdupq_n_f32(0.0f);+ const float32x4_t neg_x = vsubq_f32(zero, x);+ const float32x4_t exp_neg_x = ggml_v_expf(neg_x);+ const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x);+ return vdivq_f32(x, one_plus_exp_neg_x);+}++#elif defined(__AVX512F__) && defined(__AVX512DQ__)++// adapted from arm limited optimized routine+// the maximum error is 1.45358 plus 0.5 ulps+// numbers above 88.38 will flush to infinity+// numbers beneath -103.97 will flush to zero+inline static __m512 ggml_v_expf(__m512 x) {+ const __m512 r = _mm512_set1_ps(0x1.8p23f);+ const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r);+ const __m512 n = _mm512_sub_ps(z, r);+ const __m512 b =+ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f),+ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x));+ const __mmask16 d =+ _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(192), _CMP_GT_OQ);+ const __m512 u = _mm512_mul_ps(b, b);+ const __m512 j = _mm512_fmadd_ps(+ _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b,+ _mm512_set1_ps(0x1.573e2ep-5f)),+ u,+ _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b,+ _mm512_set1_ps(0x1.fffdb6p-2f))),+ u,+ _mm512_fmadd_ps(_mm512_set1_ps(0x1.ffffecp-1f), b, _mm512_set1_ps(1.0F)));+ const __m512 res = _mm512_scalef_ps(j, n);+ if (_mm512_kortestz(d, d))+ return res;+ const __m512 zero = _mm512_setzero_ps();+ const __m512 alt = _mm512_mask_blend_ps(+ _mm512_cmp_ps_mask(n, zero, _CMP_LE_OQ), _mm512_set1_ps(INFINITY), zero);+ return _mm512_mask_blend_ps(d, res, alt);+}++// computes silu x/(1+exp(-x)) in single precision vector+inline static __m512 ggml_v_silu(__m512 x) {+ const __m512 one = _mm512_set1_ps(1);+ const __m512 zero = _mm512_setzero_ps();+ const __m512 neg_x = _mm512_sub_ps(zero, x);+ const __m512 exp_neg_x = ggml_v_expf(neg_x);+ const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x);+ return _mm512_div_ps(x, one_plus_exp_neg_x);+}++#elif defined(__AVX2__) && defined(__FMA__)++// adapted from arm limited optimized routine+// the maximum error is 1.45358 plus 0.5 ulps+// numbers above 88.38 will flush to infinity+// numbers beneath -103.97 will flush to zero+inline static __m256 ggml_v_expf(__m256 x) {+ const __m256 r = _mm256_set1_ps(0x1.8p23f);+ const __m256 z = _mm256_fmadd_ps(x, _mm256_set1_ps(0x1.715476p+0f), r);+ const __m256 n = _mm256_sub_ps(z, r);+ const __m256 b = _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.7f7d1cp-20f),+ _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.62e4p-1f), x));+ const __m256i e = _mm256_slli_epi32(_mm256_castps_si256(z), 23);+ const __m256 k = _mm256_castsi256_ps(+ _mm256_add_epi32(e, _mm256_castps_si256(_mm256_set1_ps(1))));+ const __m256i c = _mm256_castps_si256(+ _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),+ _mm256_set1_ps(126), _CMP_GT_OQ));+ const __m256 u = _mm256_mul_ps(b, b);+ const __m256 j = _mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), b,+ _mm256_set1_ps(0x1.573e2ep-5f)), u,+ _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), b,+ _mm256_set1_ps(0x1.fffdb6p-2f))),+ u, _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), b));+ if (!_mm256_movemask_ps(_mm256_castsi256_ps(c)))+ return _mm256_fmadd_ps(j, k, k);+ const __m256i g = _mm256_and_si256(+ _mm256_castps_si256(_mm256_cmp_ps(n, _mm256_setzero_ps(), _CMP_LE_OQ)),+ _mm256_set1_epi32(0x82000000u));+ const __m256 s1 =+ _mm256_castsi256_ps(_mm256_add_epi32(g, _mm256_set1_epi32(0x7f000000u)));+ const __m256 s2 = _mm256_castsi256_ps(_mm256_sub_epi32(e, g));+ const __m256i d = _mm256_castps_si256(+ _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),+ _mm256_set1_ps(192), _CMP_GT_OQ));+ return _mm256_or_ps(+ _mm256_and_ps(_mm256_castsi256_ps(d), _mm256_mul_ps(s1, s1)),+ _mm256_andnot_ps(+ _mm256_castsi256_ps(d),+ _mm256_or_ps(+ _mm256_and_ps(_mm256_castsi256_ps(c),+ _mm256_mul_ps(_mm256_fmadd_ps(s2, j, s2), s1)),+ _mm256_andnot_ps(_mm256_castsi256_ps(c), _mm256_fmadd_ps(k, j, k)))));+}++// computes silu x/(1+exp(-x)) in single precision vector+inline static __m256 ggml_v_silu(__m256 x) {+ const __m256 one = _mm256_set1_ps(1);+ const __m256 zero = _mm256_setzero_ps();+ const __m256 neg_x = _mm256_sub_ps(zero, x);+ const __m256 exp_neg_x = ggml_v_expf(neg_x);+ const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x);+ return _mm256_div_ps(x, one_plus_exp_neg_x);+}++#elif defined(__SSE2__) // __AVX2__ / __ARM_NEON++#if defined(__FMA__)+#define MADD128(x, y, z) _mm_fmadd_ps(x, y, z)+#define NMADD128(x, y, z) _mm_fnmadd_ps(x, y, z)+#else+#define MADD128(x, y, z) _mm_add_ps(_mm_mul_ps(x, y), z)+#define NMADD128(x, y, z) _mm_sub_ps(z, _mm_mul_ps(x, y))+#endif++// adapted from arm limited optimized routine+// the maximum error is 1.45358 plus 0.5 ulps+// numbers above 88.38 will flush to infinity+// numbers beneath -103.97 will flush to zero+inline static __m128 ggml_v_expf(__m128 x) {+ const __m128 r = _mm_set1_ps(0x1.8p23f);+ const __m128 z = MADD128(x, _mm_set1_ps(0x1.715476p+0f), r);+ const __m128 n = _mm_sub_ps(z, r);+ const __m128 b =+ NMADD128(n, _mm_set1_ps(0x1.7f7d1cp-20f), NMADD128(n, _mm_set1_ps(0x1.62e4p-1f), x));+ const __m128i e = _mm_slli_epi32(_mm_castps_si128(z), 23);+ const __m128 k = _mm_castsi128_ps(_mm_add_epi32(e, _mm_castps_si128(_mm_set1_ps(1))));+ const __m128i c =+ _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(126)));+ const __m128 u = _mm_mul_ps(b, b);+ const __m128 j =+ MADD128(MADD128(MADD128(_mm_set1_ps(0x1.0e4020p-7f), b, _mm_set1_ps(0x1.573e2ep-5f)), u,+ MADD128(_mm_set1_ps(0x1.555e66p-3f), b, _mm_set1_ps(0x1.fffdb6p-2f))),+ u, _mm_mul_ps(_mm_set1_ps(0x1.ffffecp-1f), b));+ if (!_mm_movemask_epi8(c))+ return MADD128(j, k, k);+ const __m128i g = _mm_and_si128(_mm_castps_si128(_mm_cmple_ps(n, _mm_setzero_ps())),+ _mm_set1_epi32(0x82000000u));+ const __m128 s1 = _mm_castsi128_ps(_mm_add_epi32(g, _mm_set1_epi32(0x7f000000u)));+ const __m128 s2 = _mm_castsi128_ps(_mm_sub_epi32(e, g));+ const __m128i d =+ _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(192)));+ return _mm_or_ps(+ _mm_and_ps(_mm_castsi128_ps(d), _mm_mul_ps(s1, s1)),+ _mm_andnot_ps(_mm_castsi128_ps(d),+ _mm_or_ps(_mm_and_ps(_mm_castsi128_ps(c), _mm_mul_ps(MADD128(s2, j, s2), s1)),+ _mm_andnot_ps(_mm_castsi128_ps(c), MADD128(k, j, k)))));+}++// computes silu x/(1+exp(-x)) in single precision vector+inline static __m128 ggml_v_silu(__m128 x) {+ const __m128 one = _mm_set1_ps(1);+ const __m128 zero = _mm_setzero_ps();+ const __m128 neg_x = _mm_sub_ps(zero, x);+ const __m128 exp_neg_x = ggml_v_expf(neg_x);+ const __m128 one_plus_exp_neg_x = _mm_add_ps(one, exp_neg_x);+ return _mm_div_ps(x, one_plus_exp_neg_x);+}++#elif defined(__riscv_v_intrinsic)++// adapted from arm limited optimized routine+// the maximum error is 1.45358 plus 0.5 ulps+// numbers above 88.38 will flush to infinity+// numbers beneath -103.97 will flush to zero+inline static vfloat32m2_t ggml_v_expf_m2(vfloat32m2_t x, int vl) {+ const vfloat32m2_t r = __riscv_vfmv_v_f_f32m2(0x1.8p23f, vl);+#ifdef __riscv_xtheadvector+ // workaround for compiler bug (gcc 14.3.0: Error: unrecognized opcode `th.vmv1r.v v2,v4')+ vfloat32m2_t z = __riscv_vfadd_vf_f32m2(r, 0.0f, vl);+ z = __riscv_vfmacc_vf_f32m2(z, 0x1.715476p+0f, x, vl);+#else+ const vfloat32m2_t z = __riscv_vfmacc_vf_f32m2(r, 0x1.715476p+0f, x, vl);+#endif+ const vfloat32m2_t n = __riscv_vfsub_vv_f32m2(z, r, vl);+ const vfloat32m2_t b = __riscv_vfnmsac_vf_f32m2(__riscv_vfnmsac_vf_f32m2(x, 0x1.62e4p-1f, n, vl),+ 0x1.7f7d1cp-20f, n, vl);+ const vuint32m2_t e = __riscv_vsll_vx_u32m2(__riscv_vreinterpret_v_f32m2_u32m2(z), 23, vl);+ const vfloat32m2_t k = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(e, 0x3f800000, vl)); // 1.0f+ const vbool16_t c = __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 126.0f, vl);+ const vfloat32m2_t u = __riscv_vfmul_vv_f32m2(b, b, vl);+ const vfloat32m2_t j = __riscv_vfmacc_vv_f32m2(+ __riscv_vfmul_vf_f32m2(b, 0x1.ffffecp-1f, vl),+ __riscv_vfmacc_vv_f32m2(+ __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.fffdb6p-2f, vl), 0x1.555e66p-3f, b, vl),+ __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.573e2ep-5f, vl), 0x1.0e4020p-7f, b, vl),+ u, vl), u, vl);+ if (!__riscv_vcpop_m_b16(c, vl))+ return __riscv_vfmacc_vv_f32m2(k, j, k, vl);+ const vbool16_t dm = __riscv_vmfle_vf_f32m2_b16(n, 0.0f, vl);+ const vuint32m2_t d = __riscv_vmerge_vxm_u32m2(__riscv_vmv_v_x_u32m2(0, vl), 0x82000000, dm, vl);+ const vfloat32m2_t s1 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(d, 0x7f000000, vl));+ const vfloat32m2_t s2 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vsub_vv_u32m2(e, d, vl));+ const vfloat32m2_t r1 = __riscv_vmerge_vvm_f32m2(+ __riscv_vfmacc_vv_f32m2(k, k, j, vl),+ __riscv_vfmul_vv_f32m2(__riscv_vfmacc_vv_f32m2(s2, s2, j, vl), s1, vl),+ c, vl);+ return __riscv_vmerge_vvm_f32m2(+ r1, __riscv_vfmul_vv_f32m2(s1, s1, vl),+ __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 192.0f, vl),+ vl);+}++// computes silu x/(1+exp(-x)) in single precision vector+inline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) {+ const vfloat32m2_t neg_x = __riscv_vfneg_v_f32m2(x, vl);+ const vfloat32m2_t exp_neg_x = ggml_v_expf_m2(neg_x, vl);+ const vfloat32m2_t one_plus_exp_neg_x = __riscv_vfadd_vf_f32m2(exp_neg_x, 1.0f, vl);+ return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl);+}++#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic++inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_silu_f16(x[i]);+ }+}++inline static void ggml_vec_silu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_BF16(ggml_silu_f32(GGML_BF16_TO_FP32(x[i])));+ }+}++inline static float ggml_silu_backward_f32(float x, float dy) {+ const float s = 1.0f/(1.0f + expf(-x));+ return dy*s*(1.0f + x*(1.0f - s));+}++inline static ggml_fp16_t ggml_silu_backward_f16(ggml_fp16_t x, ggml_fp16_t dy) {+ const float v = GGML_CPU_FP16_TO_FP32(x);+ const float s = 1.0f/(1.0f + expf(-v));+ return GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(dy)*s*(1.0f + v*(1.0f - s)));+}++inline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {+ for (int i = 0; i < n; ++i) {+ dx[i] = ggml_silu_backward_f32(x[i], dy[i]);+ }+}++inline static void ggml_vec_silu_backward_f16(const int n, ggml_fp16_t * dx, const ggml_fp16_t * x, const ggml_fp16_t * dy) {+ for (int i = 0; i < n; ++i) {+ dx[i] = ggml_silu_backward_f16(x[i], dy[i]);+ }+}++inline static void ggml_vec_reglu_f32 (const int n, float * y, const float * x, const float * g) {+ for (int i = 0; i < n; ++i) {+ y[i] = (x[i] > 0.f) ? x[i] * g[i] : 0.f;+ }+}++inline static void ggml_vec_reglu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(x[i]);+ y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v * GGML_CPU_FP16_TO_FP32(g[i]) : 0.f);+ }+}++inline static void ggml_vec_reglu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) {+ for (int i = 0; i < n; ++i) {+ float v = GGML_BF16_TO_FP32(x[i]);+ y[i] = GGML_FP32_TO_BF16((v > 0.f) ? v * GGML_BF16_TO_FP32(g[i]) : 0.f);+ }+}++#ifdef GGML_GELU_FP16+inline static void ggml_vec_geglu_f32(const int n, float * y, const float * x, const float * g) {+ uint16_t t;+ for (int i = 0; i < n; ++i) {+ if (x[i] <= -10.0f) {+ y[i] = 0.0f;+ } else if (x[i] >= 10.0f) {+ y[i] = x[i] * g[i];+ } else {+ ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);+ memcpy(&t, &fp16, sizeof(uint16_t));+ y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[t]) * g[i];+ }+ }+}+#else+inline static void ggml_vec_geglu_f32(const int n, float * y, const float * x, const float * g) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_gelu_f32(x[i]) * g[i];+ }+}+#endif++inline static void ggml_vec_geglu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {+ const uint16_t * i16 = (const uint16_t *) x;+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(g[i]);+ y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[i16[i]]) * v);+ }+}++inline static void ggml_vec_geglu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_BF16(ggml_gelu_f32(GGML_BF16_TO_FP32(x[i])) * GGML_BF16_TO_FP32(g[i]));+ }+}++void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g);++inline static void ggml_vec_swiglu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {+ for (int i = 0; i < n; ++i) {+ float xi = GGML_CPU_FP16_TO_FP32(x[i]);+ float gi = GGML_CPU_FP16_TO_FP32(g[i]);+ y[i] = GGML_CPU_FP32_TO_FP16((xi/(1.0f + expf(-xi))) * gi);+ }+}++inline static void ggml_vec_swiglu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) {+ for (int i = 0; i < n; ++i) {+ float xi = GGML_BF16_TO_FP32(x[i]);+ float gi = GGML_BF16_TO_FP32(g[i]);+ y[i] = GGML_FP32_TO_BF16((xi/(1.0f + expf(-xi))) * gi);+ }+}++inline static void ggml_vec_geglu_erf_f32(const int n, float * y, const float * x, const float * g) {+ for (int i = 0; i < n; ++i) {+ float xi = x[i];+ y[i] = 0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * g[i];+ }+}++inline static void ggml_vec_geglu_erf_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {+ for (int i = 0; i < n; ++i) {+ float xi = GGML_CPU_FP16_TO_FP32(x[i]);+ float gi = GGML_CPU_FP16_TO_FP32(g[i]);+ y[i] = GGML_CPU_FP32_TO_FP16(0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * gi);+ }+}++inline static void ggml_vec_geglu_erf_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) {+ for (int i = 0; i < n; ++i) {+ float xi = GGML_BF16_TO_FP32(x[i]);+ float gi = GGML_BF16_TO_FP32(g[i]);+ y[i] = GGML_FP32_TO_BF16(0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * gi);+ }+}++#ifdef GGML_GELU_QUICK_FP16+inline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) {+ uint16_t t;+ for (int i = 0; i < n; ++i) {+ ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);+ memcpy(&t, &fp16, sizeof(uint16_t));+ y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[t]) * g[i];+ }+}+#else+inline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_gelu_quick_f32(x[i]) * g[i];+ }+}+#endif++inline static void ggml_vec_geglu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {+ const uint16_t * i16 = (const uint16_t *) x;+ for (int i = 0; i < n; ++i) {+ float v = GGML_CPU_FP16_TO_FP32(g[i]);+ y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[i16[i]]) * v);+ }+}++inline static void ggml_vec_geglu_quick_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) {+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_BF16(ggml_gelu_quick_f32(GGML_BF16_TO_FP32(x[i])) * GGML_BF16_TO_FP32(g[i]));+ }+}++inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {+#ifndef GGML_USE_ACCELERATE+ ggml_float sum = 0.0;+ for (int i = 0; i < n; ++i) {+ sum += (ggml_float)x[i];+ }+ *s = (float)sum;+#else+ vDSP_sve(x, 1, s, n);+#endif+}++inline static void ggml_vec_cumsum_f32(const int n, float * y, const float * x) {+ for (int i = 0; i < n; ++i) {+ if (i == 0) {+ y[i] = x[i];+ } else {+ y[i] = y[i - 1] + x[i];+ }+ }+}++inline static void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {+ ggml_float sum = 0.0;+ for (int i = 0; i < n; ++i) {+ sum += (ggml_float)x[i];+ }+ *s = sum;+}++inline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {+ float sum = 0.0f;+ for (int i = 0; i < n; ++i) {+ sum += GGML_CPU_FP16_TO_FP32(x[i]);+ }+ *s = sum;+}++inline static void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {+ float sum = 0.0f;+ for (int i = 0; i < n; ++i) {+ sum += GGML_BF16_TO_FP32(x[i]);+ }+ *s = sum;+}++inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {+#ifndef GGML_USE_ACCELERATE+ float max = -INFINITY;+ for (int i = 0; i < n; ++i) {+ max = MAX(max, x[i]);+ }+ *s = max;+#else+ vDSP_maxv(x, 1, s, n);+#endif+}++inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {+ ggml_vec_norm_f32(n, s, x);+ *s = 1.f/(*s);+}++inline static void ggml_vec_argmax_f32(const int n, int * s, const float * x) {+ float max = -INFINITY;+ int idx = 0;+ for (int i = 0; i < n; ++i) {+ max = MAX(max, x[i]);+ if (max == x[i]) { idx = i; }+ }+ *s = idx;+}++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/include/ggml-cpu.h b/python/freetoken/kernel/csrc/ggml/include/ggml-cpu.hnew file mode 100644index 00000000..dc6453c6--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/include/ggml-cpu.h@@ -0,0 +1,152 @@+#pragma once++#include "ggml.h"+#include "ggml-backend.h"++#ifdef __cplusplus+extern "C" {+#endif++ // the compute plan that needs to be prepared for ggml_graph_compute()+ // since https://github.com/ggml-org/ggml/issues/287+ struct ggml_cplan {+ size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()`+ uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()`++ int n_threads;+ struct ggml_threadpool * threadpool;++ // abort ggml_graph_compute when true+ ggml_abort_callback abort_callback;+ void * abort_callback_data;++ // use only reference implementations+ bool use_ref;+ };++ // numa strategies+ enum ggml_numa_strategy {+ GGML_NUMA_STRATEGY_DISABLED = 0,+ GGML_NUMA_STRATEGY_DISTRIBUTE = 1,+ GGML_NUMA_STRATEGY_ISOLATE = 2,+ GGML_NUMA_STRATEGY_NUMACTL = 3,+ GGML_NUMA_STRATEGY_MIRROR = 4,+ GGML_NUMA_STRATEGY_COUNT+ };++ GGML_BACKEND_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems+ GGML_BACKEND_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node++ GGML_BACKEND_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);+ GGML_BACKEND_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);++ GGML_BACKEND_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);+ GGML_BACKEND_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);++ GGML_BACKEND_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);+ GGML_BACKEND_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);++ GGML_BACKEND_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);+ GGML_BACKEND_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);++ GGML_BACKEND_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);+ GGML_BACKEND_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);++ GGML_BACKEND_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);+ GGML_BACKEND_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);++ GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params);+ GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool);+ GGML_BACKEND_API int ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool);+ GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool);+ GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool);++ // ggml_graph_plan() has to be called before ggml_graph_compute()+ // when plan.work_size > 0, caller must allocate memory for plan.work_data+ GGML_BACKEND_API struct ggml_cplan ggml_graph_plan(+ const struct ggml_cgraph * cgraph,+ int n_threads, /* = GGML_DEFAULT_N_THREADS */+ struct ggml_threadpool * threadpool /* = NULL */ );+ GGML_BACKEND_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);++ // same as ggml_graph_compute() but the work data is allocated as a part of the context+ // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data+ GGML_BACKEND_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);++ //+ // system info+ //++ // x86+ GGML_BACKEND_API int ggml_cpu_has_sse3 (void);+ GGML_BACKEND_API int ggml_cpu_has_ssse3 (void);+ GGML_BACKEND_API int ggml_cpu_has_avx (void);+ GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void);+ GGML_BACKEND_API int ggml_cpu_has_avx2 (void);+ GGML_BACKEND_API int ggml_cpu_has_bmi2 (void);+ GGML_BACKEND_API int ggml_cpu_has_f16c (void);+ GGML_BACKEND_API int ggml_cpu_has_fma (void);+ GGML_BACKEND_API int ggml_cpu_has_avx512 (void);+ GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void);+ GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void);+ GGML_BACKEND_API int ggml_cpu_has_avx512_bf16(void);+ GGML_BACKEND_API int ggml_cpu_has_amx_int8 (void);+ // ARM+ GGML_BACKEND_API int ggml_cpu_has_neon (void);+ GGML_BACKEND_API int ggml_cpu_has_arm_fma (void);+ GGML_BACKEND_API int ggml_cpu_has_fp16_va (void);+ GGML_BACKEND_API int ggml_cpu_has_dotprod (void);+ GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void);+ GGML_BACKEND_API int ggml_cpu_has_sve (void);+ GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytes+ GGML_BACKEND_API int ggml_cpu_has_sme (void);+ GGML_BACKEND_API int ggml_cpu_has_sme2 (void);+ // other+ GGML_BACKEND_API int ggml_cpu_has_riscv_v (void);+ GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes+ GGML_BACKEND_API int ggml_cpu_has_vsx (void);+ GGML_BACKEND_API int ggml_cpu_has_vxe (void);+ GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void);+ GGML_BACKEND_API int ggml_cpu_has_llamafile (void);++ // Internal types and functions exposed for tests and benchmarks++ typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,+ const void * GGML_RESTRICT y, size_t by, int nrc);++ struct ggml_type_traits_cpu {+ ggml_from_float_t from_float;+ ggml_vec_dot_t vec_dot;+ enum ggml_type vec_dot_type;+ int64_t nrows; // number of rows to process simultaneously+ };++ GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type);++ GGML_BACKEND_API void ggml_cpu_init(void);++ //+ // CPU backend+ //++ GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void);++ GGML_BACKEND_API bool ggml_backend_is_cpu (ggml_backend_t backend);+ GGML_BACKEND_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);+ GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);+ GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);++ GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref);++ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void);++ GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *, float *, int64_t);+ GGML_BACKEND_API void ggml_cpu_fp32_to_i32 (const float *, int32_t *, int64_t);+ GGML_BACKEND_API void ggml_cpu_fp32_to_fp16(const float *, ggml_fp16_t *, int64_t);+ GGML_BACKEND_API void ggml_cpu_fp16_to_fp32(const ggml_fp16_t *, float *, int64_t);+ GGML_BACKEND_API void ggml_cpu_fp32_to_bf16(const float *, ggml_bf16_t *, int64_t);+ GGML_BACKEND_API void ggml_cpu_bf16_to_fp32(const ggml_bf16_t *, float *, int64_t);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/src/ggml-backend-impl.h b/python/freetoken/kernel/csrc/ggml/src/ggml-backend-impl.hnew file mode 100644index 00000000..dc7367de--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/src/ggml-backend-impl.h@@ -0,0 +1,292 @@+#pragma once++// ggml-backend internal header++#include "ggml-backend.h"++#ifdef __cplusplus+extern "C" {+#endif++ #define GGML_BACKEND_API_VERSION 3++ //+ // Backend buffer type+ //++ struct ggml_backend_buffer_type_i {+ const char * (*get_name) (ggml_backend_buffer_type_t buft);+ // allocate a buffer of this type+ ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);+ // (optional) allocate tensors from a list into a buffer of this type (defaults to alloc_buffer + linear allocator)+ ggml_backend_buffer_t (*alloc_buffer_n) (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);+ // tensor alignment+ size_t (*get_alignment) (ggml_backend_buffer_type_t buft);+ // (optional) max buffer size that can be allocated (defaults to SIZE_MAX)+ size_t (*get_max_size) (ggml_backend_buffer_type_t buft);+ // (optional) data size needed to allocate the tensor, including padding (defaults to ggml_nbytes)+ size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);+ // (optional) total data size needed to allocate the given tensors, including padding and splitting (defaults to per-tensor get_alloc_size)+ size_t (*get_alloc_size_n)(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);+ // (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false)+ bool (*is_host) (ggml_backend_buffer_type_t buft);+ };++ struct ggml_backend_buffer_type {+ struct ggml_backend_buffer_type_i iface;+ ggml_backend_dev_t device;+ void * context;+ };++ // [TAG_ALLOC_SIZE_EXPAND]+ // returns true for ops that may require additional memory for fleeting data on some backends,+ // i.e. the backend buffer type's get_alloc_size may return more than ggml_nbytes for the output tensor+ GGML_API bool ggml_op_alloc_size_may_expand(enum ggml_op op);++ //+ // Backend buffer+ //++ struct ggml_backend_buffer_i {+ // (optional) free the buffer+ void (*free_buffer) (ggml_backend_buffer_t buffer);+ // base address of the buffer+ void * (*get_base) (ggml_backend_buffer_t buffer);+ // (optional) initialize a tensor in the buffer (eg. add tensor extras)+ enum ggml_status (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ // tensor data access+ void (*memset_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);+ void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ // (optional) 2d data copies+ void (*set_tensor_2d)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+ void (*get_tensor_2d)(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);++ // (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)+ bool (*cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst);+ // clear the entire buffer+ void (*clear) (ggml_backend_buffer_t buffer, uint8_t value);+ // (optional) reset any internal state due to tensor initialization, such as tensor extras+ void (*reset) (ggml_backend_buffer_t buffer);+ };++ struct ggml_backend_buffer {+ struct ggml_backend_buffer_i iface;+ ggml_backend_buffer_type_t buft;+ void * context;+ size_t size;+ enum ggml_backend_buffer_usage usage;+ };++ GGML_API ggml_backend_buffer_t ggml_backend_buffer_init(+ ggml_backend_buffer_type_t buft,+ struct ggml_backend_buffer_i iface,+ void * context,+ size_t size);++ // do not use directly, use ggml_backend_tensor_copy instead+ GGML_API bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);++ // multi-buffer+ // buffer that contains a collection of buffers+ GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);+ GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);+ GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);+ GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);++ //+ // Backend (meta)+ //++ GGML_API bool ggml_backend_is_meta (ggml_backend_t backend);+ GGML_API bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf);+ GGML_API bool ggml_backend_buft_is_meta (ggml_backend_buffer_type_t buft);++ GGML_API size_t ggml_backend_meta_n_backends (ggml_backend_t meta_backend);+ GGML_API ggml_backend_t ggml_backend_meta_simple_backend(ggml_backend_t meta_backend, size_t index);++ //+ // Backend (stream)+ //++ // passed to graph_optimize so the backend can add allocation dependencies:+ // if the backend executes parts of the graph out of order (e.g. on concurrent streams),+ // it must keep the affected tensors allocated until a node where execution is known to have joined+ struct ggml_backend_graph_optimize_params {+ // keep `tensor` allocated at least until `until` (a node of the same graph) has been computed+ // can be called multiple times for the same tensor: the longest lifetime applies+ void (*add_alloc_dep)(void * user_data, struct ggml_tensor * tensor, struct ggml_tensor * until);+ void * user_data;+ };++ struct ggml_backend_i {+ const char * (*get_name)(ggml_backend_t backend);++ void (*free)(ggml_backend_t backend);++ // (optional) asynchronous tensor data access+ void (*set_tensor_async) (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ void (*get_tensor_async) (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ void (*set_tensor_2d_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+ void (*get_tensor_2d_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+ bool (*cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);++ // (optional) complete all pending operations (required if the backend supports async operations)+ void (*synchronize)(ggml_backend_t backend);++ // (optional) graph plans (not used currently)+ // compute graph with a plan+ ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);+ void (*graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);+ // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology+ void (*graph_plan_update) (ggml_backend_t backend, ggml_backend_graph_plan_t plan, const struct ggml_cgraph * cgraph);+ // compute the graph with the plan+ enum ggml_status (*graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);++ // compute graph (always async if supported by the backend)+ enum ggml_status (*graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph);++ // (optional) event synchronization+ // record an event on this stream+ void (*event_record)(ggml_backend_t backend, ggml_backend_event_t event);+ // wait for an event on on a different stream+ void (*event_wait) (ggml_backend_t backend, ggml_backend_event_t event);++ // (optional) sort/optimize the nodes in the graph+ void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params);+ };++ struct ggml_backend {+ ggml_guid_t guid;+ struct ggml_backend_i iface;+ ggml_backend_dev_t device;+ void * context;+ };++ struct ggml_backend_event {+ struct ggml_backend_device * device;+ void * context;+ };++ //+ // Backend device+ //++ // Note: if additional properties are needed, we should add a struct with all of them+ // the current functions to obtain the properties can remain, since they are more convenient for often used properties+ struct ggml_backend_device_i {+ // device name: short identifier for this device, such as "CPU" or "CUDA0"+ const char * (*get_name)(ggml_backend_dev_t dev);++ // device description: short informative description of the device, could be the model name+ const char * (*get_description)(ggml_backend_dev_t dev);++ // device memory in bytes: 0 bytes to indicate no memory to report+ void (*get_memory)(ggml_backend_dev_t dev, size_t * free, size_t * total);++ // device type+ enum ggml_backend_dev_type (*get_type)(ggml_backend_dev_t dev);++ // device properties+ void (*get_props)(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props);++ // backend (stream) initialization+ ggml_backend_t (*init_backend)(ggml_backend_dev_t dev, const char * params);++ // preferred buffer type+ ggml_backend_buffer_type_t (*get_buffer_type)(ggml_backend_dev_t dev);++ // (optional) host buffer type (in system memory, typically this is a pinned memory buffer for faster transfers between host and device)+ ggml_backend_buffer_type_t (*get_host_buffer_type)(ggml_backend_dev_t dev);++ // (optional) buffer from pointer: create a buffer from a host pointer (useful for memory mapped models and importing data from other libraries)+ ggml_backend_buffer_t (*buffer_from_host_ptr)(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size);++ // check if the backend can compute an operation+ bool (*supports_op)(ggml_backend_dev_t dev, const struct ggml_tensor * op);++ // check if the backend can use tensors allocated in a buffer type+ bool (*supports_buft)(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft);++ // (optional) check if the backend wants to run an operation, even if the weights are allocated in an incompatible buffer+ // these should be expensive operations that may benefit from running on this backend instead of the CPU backend+ bool (*offload_op)(ggml_backend_dev_t dev, const struct ggml_tensor * op);++ // (optional) event synchronization+ ggml_backend_event_t (*event_new) (ggml_backend_dev_t dev);+ void (*event_free) (ggml_backend_dev_t dev, ggml_backend_event_t event);+ void (*event_synchronize) (ggml_backend_dev_t dev, ggml_backend_event_t event);+ };++ struct ggml_backend_device {+ struct ggml_backend_device_i iface;+ ggml_backend_reg_t reg;+ void * context;+ };++ //+ // Backend (reg)+ //++ struct ggml_backend_reg_i {+ const char * (*get_name)(ggml_backend_reg_t reg);++ // enumerate available devices+ size_t (*get_device_count)(ggml_backend_reg_t reg);+ ggml_backend_dev_t (*get_device)(ggml_backend_reg_t reg, size_t index);++ // (optional) get a pointer to a function in the backend+ // backends can add custom functions that are not part of the standard ggml-backend interface+ void * (*get_proc_address)(ggml_backend_reg_t reg, const char * name);+ };++ struct ggml_backend_reg {+ int api_version; // initialize to GGML_BACKEND_API_VERSION+ struct ggml_backend_reg_i iface;+ void * context;+ };++ // Add backend dynamic loading support to the backend++ // Initialize the backend+ typedef ggml_backend_reg_t (*ggml_backend_init_t)(void);+ // Optional: obtain a score for the backend based on the system configuration+ // Higher scores are preferred, 0 means the backend is not supported in the current system+ typedef int (*ggml_backend_score_t)(void);++#ifdef GGML_BACKEND_DL+# ifdef __cplusplus+# define GGML_BACKEND_DL_IMPL(reg_fn) \+ extern "C" { \+ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \+ } \+ ggml_backend_reg_t ggml_backend_init(void) { \+ return reg_fn(); \+ }+# define GGML_BACKEND_DL_SCORE_IMPL(score_fn) \+ extern "C" { \+ GGML_BACKEND_API int ggml_backend_score(void); \+ } \+ int ggml_backend_score(void) { \+ return score_fn(); \+ }+# else+# define GGML_BACKEND_DL_IMPL(reg_fn) \+ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \+ ggml_backend_reg_t ggml_backend_init(void) { \+ return reg_fn(); \+ }+# define GGML_BACKEND_DL_SCORE_IMPL(score_fn) \+ GGML_BACKEND_API int ggml_backend_score(void); \+ int ggml_backend_score(void) { \+ return score_fn(); \+ }+# endif+#else+# define GGML_BACKEND_DL_IMPL(reg_fn)+# define GGML_BACKEND_DL_SCORE_IMPL(score_fn)+#endif++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/src/ggml-quants.c b/python/freetoken/kernel/csrc/ggml/src/ggml-quants.cnew file mode 100644index 00000000..7750a72c--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/src/ggml-quants.c@@ -0,0 +1,5639 @@+#define GGML_COMMON_IMPL_C+#include "ggml-common.h"++#include "ggml-quants.h"+#include "ggml-impl.h"+#include "ggml-cpu/ggml-cpu-impl.h"+#include "ggml-cpu.h"++#include <math.h>+#include <string.h>+#include <assert.h>+#include <float.h>+#include <stdlib.h> // for qsort+#include <stdio.h> // for GGML_ASSERT++#ifdef GGML_USE_OPENMP+#include <omp.h>+#endif++#define GROUP_MAX_EPS 1e-15f+#define GROUP_MAX_EPS_IQ3_XXS 1e-8f+#define GROUP_MAX_EPS_IQ2_S 1e-8f+#define GROUP_MAX_EPS_IQ1_M 1e-7f+#define GROUP_MAX_EPS_IQ1_S 1e-12f++#define UNUSED GGML_UNUSED++static inline int best_index_int8(int n, const int8_t * val, float x) {+ if (x <= val[0]) return 0;+ if (x >= val[n-1]) return n-1;+ int ml = 0, mu = n-1;+ while (mu-ml > 1) {+ int mav = (ml+mu)/2;+ if (x < val[mav]) mu = mav; else ml = mav;+ }+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;+}++// reference implementation for deterministic creation of model files+void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK1_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ float sum_abs = 0.0f;+ for (int j = 0; j < qk; j++) {+ sum_abs += fabsf(x[i*qk + j]);+ }+ const float d = sum_abs / qk;++ y[i].d = GGML_FP32_TO_FP16(d);++ // Clear all bits first+ for (int j = 0; j < qk / 8; ++j) {+ y[i].qs[j] = 0;+ }++ // Just store sign of each weight directly (no normalization)+ for (int j = 0; j < qk; ++j) {+ const int bit_index = j;+ const int byte_index = bit_index / 8;+ const int bit_offset = bit_index % 8;++ if (x[i*qk + j] >= 0.0f) {+ y[i].qs[byte_index] |= (1 << bit_offset);+ }+ }+ }+}++void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK2_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ // Compute scale as max absolute value in the block+ float amax = 0.0f;+ for (int j = 0; j < qk; j++) {+ const float a = fabsf(x[i*qk + j]);+ if (a > amax) amax = a;+ }+ const float d = amax;+ const float id = d > 0.0f ? 1.0f / d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ // Clear quant bytes+ for (int j = 0; j < qk / 4; ++j) {+ y[i].qs[j] = 0;+ }++ // Encode 2-bit values: round(w/d) clamped to [-1, 2], then add 1+ // 00 (-1) = -scale, 01 (0) = 0, 10 (+1) = +scale, 11 (+2) = 2*scale+ for (int j = 0; j < qk; ++j) {+ const float w = x[i*qk + j];+ int q = (int)roundf(w * id) + 1;+ if (q < 0) q = 0;+ if (q > 3) q = 3;+ const int byte_index = j / 4;+ const int bit_offset = (j % 4) * 2;+ y[i].qs[byte_index] |= ((uint8_t)q << bit_offset);+ }+ }+}++// reference implementation for deterministic creation of model files+void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK4_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max+ float max = 0.0f;++ for (int j = 0; j < qk; j++) {+ const float v = x[i*qk + j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ max = v;+ }+ }++ const float d = max / -8;+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ for (int j = 0; j < qk/2; ++j) {+ const float x0 = x[i*qk + 0 + j]*id;+ const float x1 = x[i*qk + qk/2 + j]*id;++ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));++ y[i].qs[j] = xi0;+ y[i].qs[j] |= xi1 << 4;+ }+ }+}++void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k) {+ const int qk = QK4_1;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ float min = FLT_MAX;+ float max = -FLT_MAX;++ for (int j = 0; j < qk; j++) {+ const float v = x[i*qk + j];++ if (v < min) min = v;+ if (v > max) max = v;+ }++ const float d = (max - min) / ((1 << 4) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);+ y[i].m = GGML_FP32_TO_FP16(min);++ for (int j = 0; j < qk/2; ++j) {+ const float x0 = (x[i*qk + 0 + j] - min)*id;+ const float x1 = (x[i*qk + qk/2 + j] - min)*id;++ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));++ y[i].qs[j] = xi0;+ y[i].qs[j] |= xi1 << 4;+ }+ }+}++void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK5_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max+ float max = 0.0f;++ for (int j = 0; j < qk; j++) {+ const float v = x[i*qk + j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ max = v;+ }+ }++ const float d = max / -16;+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ uint32_t qh = 0;++ for (int j = 0; j < qk/2; ++j) {+ const float x0 = x[i*qk + 0 + j]*id;+ const float x1 = x[i*qk + qk/2 + j]*id;++ const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f));+ const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f));++ y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);++ // get the 5-th bit and store it in qh at the right position+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2);+ }++ memcpy(&y[i].qh, &qh, sizeof(qh));+ }+}++void quantize_row_q5_1_ref(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k) {+ const int qk = QK5_1;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ float min = FLT_MAX;+ float max = -FLT_MAX;++ for (int j = 0; j < qk; j++) {+ const float v = x[i*qk + j];++ if (v < min) min = v;+ if (v > max) max = v;+ }++ const float d = (max - min) / ((1 << 5) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);+ y[i].m = GGML_FP32_TO_FP16(min);++ uint32_t qh = 0;++ for (int j = 0; j < qk/2; ++j) {+ const float x0 = (x[i*qk + 0 + j] - min)*id;+ const float x1 = (x[i*qk + qk/2 + j] - min)*id;++ const uint8_t xi0 = (uint8_t)(x0 + 0.5f);+ const uint8_t xi1 = (uint8_t)(x1 + 0.5f);++ y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);++ // get the 5-th bit and store it in qh at the right position+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2);+ }++ memcpy(&y[i].qh, &qh, sizeof(y[i].qh));+ }+}++// reference implementation for deterministic creation of model files+void quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k) {+ assert(k % QK8_0 == 0);+ const int nb = k / QK8_0;++ for (int i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK8_0; j++) {+ const float v = x[i*QK8_0 + j];+ amax = MAX(amax, fabsf(v));+ }++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ for (int j = 0; j < QK8_0; ++j) {+ const float x0 = x[i*QK8_0 + j]*id;++ y[i].qs[j] = roundf(x0);+ }+ }+}++// reference implementation for deterministic creation of model files+void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k) {+ assert(QK8_1 == 32);+ assert(k % QK8_1 == 0);+ const int nb = k / QK8_1;++ for (int i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK8_1; j++) {+ const float v = x[i*QK8_1 + j];+ amax = MAX(amax, fabsf(v));+ }++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ int sum = 0;++ for (int j = 0; j < QK8_1/2; ++j) {+ const float v0 = x[i*QK8_1 + j]*id;+ const float v1 = x[i*QK8_1 + QK8_1/2 + j]*id;++ y[i].qs[ j] = roundf(v0);+ y[i].qs[QK8_1/2 + j] = roundf(v1);++ sum += y[i].qs[ j];+ sum += y[i].qs[QK8_1/2 + j];+ }++ y[i].s = GGML_FP32_TO_FP16(sum*d);+ }+}++static inline int best_index_mxfp4(float x, float e) {+ int best_index = 0;+ float best_err = fabsf(kvalues_mxfp4[0]*e - x);+ for (int i = 1; i < 16; i++) {+ float err = fabsf(kvalues_mxfp4[i]*e - x);+ if (err < best_err) {+ best_index = i;+ best_err = err;+ }+ }+ return best_index;+}++void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK_MXFP4;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max++ for (int j = 0; j < qk; j++) {+ const float v = x[i*qk + j];++ if (amax < fabsf(v)) {+ amax = fabsf(v);+ }+ }++ const uint8_t e = amax > 0.0f ? (uint8_t) (floorf(log2f(amax)) - 2 + 127) : 0;++ const float d = GGML_E8M0_TO_FP32_HALF(e);++ y[i].e = e;++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t x0 = best_index_mxfp4(x[i*qk + 0 + j], d);+ const uint8_t x1 = best_index_mxfp4(x[i*qk + qk/2 + j], d);++ y[i].qs[j] = x0;+ y[i].qs[j] |= x1 << 4;+ }+ }+}++void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK_NVFP4;+ static const int qk_sub = QK_NVFP4_SUB;+ static const int n_sub = QK_NVFP4 / QK_NVFP4_SUB;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ for (int s = 0; s < n_sub; s++) {+ const float * xb = x + i*qk + s*qk_sub;++ float amax = 0.0f;+ for (int j = 0; j < qk_sub; j++) {+ if (amax < fabsf(xb[j])) {+ amax = fabsf(xb[j]);+ }+ }++ // UE4M3 scale: amax / 6.0 maps the max E2M1 value (6.0) to amax+ const uint8_t ue = ggml_fp32_to_ue4m3(amax / 6.0f);+ y[i].d[s] = ue;+ const float d = ggml_ue4m3_to_fp32(ue);++ for (int j = 0; j < qk_sub/2; ++j) {+ const uint8_t x0 = best_index_mxfp4(xb[0 + j], d);+ const uint8_t x1 = best_index_mxfp4(xb[qk_sub/2 + j], d);++ y[i].qs[s*(qk_sub/2) + j] = x0 | (x1 << 4);+ }+ }+ }+}++void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK1_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);+ const float neg_d = -d;++ for (int j = 0; j < qk; ++j) {+ const int byte_index = j / 8;+ const int bit_offset = j % 8;+ const uint8_t bit = (x[i].qs[byte_index] >> bit_offset) & 1;+ y[i*qk + j] = bit ? d : neg_d;+ }+ }+}++void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK2_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int j = 0; j < qk; ++j) {+ const int byte_index = j / 4;+ const int bit_offset = (j % 4) * 2;+ const uint8_t q = (x[i].qs[byte_index] >> bit_offset) & 0x03;+ // 00=-1, 01=0, 10=+1, 11=+2+ y[i*qk + j] = ((int)q - 1) * d;+ }+ }+}++void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK4_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int j = 0; j < qk/2; ++j) {+ const int x0 = (x[i].qs[j] & 0x0F) - 8;+ const int x1 = (x[i].qs[j] >> 4) - 8;++ y[i*qk + j + 0 ] = x0*d;+ y[i*qk + j + qk/2] = x1*d;+ }+ }+}++void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK4_1;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);+ const float m = GGML_FP16_TO_FP32(x[i].m);++ for (int j = 0; j < qk/2; ++j) {+ const int x0 = (x[i].qs[j] & 0x0F);+ const int x1 = (x[i].qs[j] >> 4);++ y[i*qk + j + 0 ] = x0*d + m;+ y[i*qk + j + qk/2] = x1*d + m;+ }+ }+}++void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK5_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);++ uint32_t qh;+ memcpy(&qh, x[i].qh, sizeof(qh));++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;++ const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;+ const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;++ y[i*qk + j + 0 ] = x0*d;+ y[i*qk + j + qk/2] = x1*d;+ }+ }+}++void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK5_1;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);+ const float m = GGML_FP16_TO_FP32(x[i].m);++ uint32_t qh;+ memcpy(&qh, x[i].qh, sizeof(qh));++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;++ const int x0 = (x[i].qs[j] & 0x0F) | xh_0;+ const int x1 = (x[i].qs[j] >> 4) | xh_1;++ y[i*qk + j + 0 ] = x0*d + m;+ y[i*qk + j + qk/2] = x1*d + m;+ }+ }+}++void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK8_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int j = 0; j < qk; ++j) {+ y[i*qk + j] = x[i].qs[j]*d;+ }+ }+}++void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK_MXFP4;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_E8M0_TO_FP32_HALF(x[i].e);++ for (int j = 0; j < qk/2; ++j) {+ const int8_t x0 = kvalues_mxfp4[x[i].qs[j] & 0x0F];+ const int8_t x1 = kvalues_mxfp4[x[i].qs[j] >> 4];++ y[i*qk + j + 0 ] = x0*d;+ y[i*qk + j + qk/2] = x1*d;+ }+ }+}++void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK_NVFP4;+ static const int qk_sub = QK_NVFP4_SUB;+ static const int n_sub = QK_NVFP4 / QK_NVFP4_SUB;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ for (int s = 0; s < n_sub; s++) {+ const float d = ggml_ue4m3_to_fp32(x[i].d[s]);+ float * yb = y + i*qk + s*qk_sub;++ for (int j = 0; j < qk_sub/2; ++j) {+ const int8_t v0 = kvalues_mxfp4[x[i].qs[s*(qk_sub/2) + j] & 0x0F];+ const int8_t v1 = kvalues_mxfp4[x[i].qs[s*(qk_sub/2) + j] >> 4];++ yb[j + 0 ] = v0*d;+ yb[j + qk_sub/2] = v1*d;+ }+ }+ }+}++//+// 2-6 bit quantization in super-blocks+//++//+// ===================== Helper functions+//+static inline int nearest_int(float fval) {+ assert(fabsf(fval) <= 4194303.f);+ float val = fval + 12582912.f;+ int i; memcpy(&i, &val, sizeof(int));+ return (i & 0x007fffff) - 0x00400000;+}++static float make_qx_quants(int n, int nmax, const float * GGML_RESTRICT x, int8_t * GGML_RESTRICT L, int rmse_type,+ const float * GGML_RESTRICT qw) {+ float max = 0;+ float amax = 0;+ for (int i = 0; i < n; ++i) {+ float ax = fabsf(x[i]);+ if (ax > amax) { amax = ax; max = x[i]; }+ }+ if (amax < GROUP_MAX_EPS) { // all zero+ for (int i = 0; i < n; ++i) {+ L[i] = 0;+ }+ return 0.f;+ }+ float iscale = -nmax / max;+ if (rmse_type == 0) {+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale * x[i]);+ L[i] = nmax + MAX(-nmax, MIN(nmax-1, l));+ }+ return 1/iscale;+ }+ bool return_early = false;+ if (rmse_type < 0) {+ rmse_type = -rmse_type;+ return_early = true;+ }+ float sumlx = 0;+ float suml2 = 0;+#ifdef HAVE_BUGGY_APPLE_LINKER+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7+ for (volatile int i = 0; i < n; ++i) {+#else+ for (int i = 0; i < n; ++i) {+#endif+ int l = nearest_int(iscale * x[i]);+ l = MAX(-nmax, MIN(nmax-1, l));+ L[i] = l + nmax;+ float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));+ sumlx += w*x[i]*l;+ suml2 += w*l*l;+ }+ float scale = suml2 ? sumlx/suml2 : 0.0f;+ if (return_early) return suml2 > 0 ? 0.5f*(scale + 1/iscale) : 1/iscale;+ float best = scale * sumlx;+ for (int is = -9; is <= 9; ++is) {+ if (is == 0) {+ continue;+ }+ iscale = -(nmax + 0.1f*is) / max;+ sumlx = suml2 = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale * x[i]);+ l = MAX(-nmax, MIN(nmax-1, l));+ float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));+ sumlx += w*x[i]*l;+ suml2 += w*l*l;+ }+ if (suml2 > 0 && sumlx*sumlx > best*suml2) {+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale * x[i]);+ L[i] = nmax + MAX(-nmax, MIN(nmax-1, l));+ }+ scale = sumlx/suml2; best = scale*sumlx;+ }+ }+ return scale;+}++static float make_q3_quants(int n, int nmax, const float * GGML_RESTRICT x, int8_t * GGML_RESTRICT L, bool do_rmse) {+ float max = 0;+ float amax = 0;+ for (int i = 0; i < n; ++i) {+ float ax = fabsf(x[i]);+ if (ax > amax) { amax = ax; max = x[i]; }+ }+ if (amax < GROUP_MAX_EPS) { // all zero+ for (int i = 0; i < n; ++i) { L[i] = 0; }+ return 0.f;+ }+ float iscale = -nmax / max;+ if (do_rmse) {+ float sumlx = 0;+ float suml2 = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale * x[i]);+ l = MAX(-nmax, MIN(nmax-1, l));+ L[i] = l;+ float w = x[i]*x[i];+ sumlx += w*x[i]*l;+ suml2 += w*l*l;+ }+ for (int itry = 0; itry < 5; ++itry) {+ int n_changed = 0;+ for (int i = 0; i < n; ++i) {+ float w = x[i]*x[i];+ float slx = sumlx - w*x[i]*L[i];+ if (slx > 0) {+ float sl2 = suml2 - w*L[i]*L[i];+ int new_l = nearest_int(x[i] * sl2 / slx);+ new_l = MAX(-nmax, MIN(nmax-1, new_l));+ if (new_l != L[i]) {+ slx += w*x[i]*new_l;+ sl2 += w*new_l*new_l;+ if (sl2 > 0 && slx*slx*suml2 > sumlx*sumlx*sl2) {+ L[i] = new_l; sumlx = slx; suml2 = sl2;+ ++n_changed;+ }+ }+ }+ }+ if (!n_changed) {+ break;+ }+ }+ for (int i = 0; i < n; ++i) {+ L[i] += nmax;+ }+ return suml2 > 0.0f ? sumlx / suml2 : 0.0f;+ }+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale * x[i]);+ l = MAX(-nmax, MIN(nmax-1, l));+ L[i] = l + nmax;+ }+ return 1/iscale;+}++static float make_qkx1_quants(int n, int nmax, const float * GGML_RESTRICT x, uint8_t * GGML_RESTRICT L, float * GGML_RESTRICT the_min,+ int ntry, float alpha) {+ float min = x[0];+ float max = x[0];+ for (int i = 1; i < n; ++i) {+ if (x[i] < min) min = x[i];+ if (x[i] > max) max = x[i];+ }+ if (max == min) {+ for (int i = 0; i < n; ++i) L[i] = 0;+ *the_min = 0;+ return 0.f;+ }+ if (min > 0) min = 0;+ float iscale = nmax/(max - min);+ float scale = 1/iscale;+ for (int itry = 0; itry < ntry; ++itry) {+ float sumlx = 0; int suml2 = 0;+ bool did_change = false;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale*(x[i] - min));+ l = MAX(0, MIN(nmax, l));+ if (l != L[i]) {+ L[i] = l;+ did_change = true;+ }+ sumlx += (x[i] - min)*l;+ suml2 += l*l;+ }+ scale = sumlx/suml2;+ float sum = 0;+ for (int i = 0; i < n; ++i) {+ sum += x[i] - scale*L[i];+ }+ min = alpha*min + (1 - alpha)*sum/n;+ if (min > 0) min = 0;+ iscale = 1/scale;+ if (!did_change) break;+ }+ *the_min = -min;+ return scale;+}++static float make_qkx2_quants(int n, int nmax, const float * GGML_RESTRICT x, const float * GGML_RESTRICT weights,+ uint8_t * GGML_RESTRICT L, float * GGML_RESTRICT the_min, uint8_t * GGML_RESTRICT Laux,+ float rmin, float rdelta, int nstep, bool use_mad) {+ float min = x[0];+ float max = x[0];+ float sum_w = weights[0];+ float sum_x = sum_w * x[0];+#ifdef HAVE_BUGGY_APPLE_LINKER+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7+ for (volatile int i = 1; i < n; ++i) {+#else+ for (int i = 1; i < n; ++i) {+#endif+ if (x[i] < min) min = x[i];+ if (x[i] > max) max = x[i];+ float w = weights[i];+ sum_w += w;+ sum_x += w * x[i];+ }+ if (min > 0) min = 0;+ if (max == min) {+ for (int i = 0; i < n; ++i) L[i] = 0;+ *the_min = -min;+ return 0.f;+ }+ float iscale = nmax/(max - min);+ float scale = 1/iscale;+ float best_error = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale*(x[i] - min));+ L[i] = MAX(0, MIN(nmax, l));+ float diff = scale * L[i] + min - x[i];+ diff = use_mad ? fabsf(diff) : diff * diff;+ float w = weights[i];+ best_error += w * diff;+ }+ if (nstep < 1) {+ *the_min = -min;+ return scale;+ }+ for (int is = 0; is <= nstep; ++is) {+ iscale = (rmin + rdelta*is + nmax)/(max - min);+ float sum_l = 0, sum_l2 = 0, sum_xl = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale*(x[i] - min));+ l = MAX(0, MIN(nmax, l));+ Laux[i] = l;+ float w = weights[i];+ sum_l += w*l;+ sum_l2 += w*l*l;+ sum_xl += w*l*x[i];+ }+ float D = sum_w * sum_l2 - sum_l * sum_l;+ if (D > 0) {+ float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D;+ float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D;+ if (this_min > 0) {+ this_min = 0;+ this_scale = sum_xl / sum_l2;+ }+ float cur_error = 0;+ for (int i = 0; i < n; ++i) {+ float diff = this_scale * Laux[i] + this_min - x[i];+ diff = use_mad ? fabsf(diff) : diff * diff;+ float w = weights[i];+ cur_error += w * diff;+ }+ if (cur_error < best_error) {+ for (int i = 0; i < n; ++i) {+ L[i] = Laux[i];+ }+ best_error = cur_error;+ scale = this_scale;+ min = this_min;+ }+ }+ }+ *the_min = -min;+ return scale;+}++static inline void get_scale_min_k4(int j, const uint8_t * GGML_RESTRICT q, uint8_t * GGML_RESTRICT d, uint8_t * GGML_RESTRICT m) {+ if (j < 4) {+ *d = q[j] & 63; *m = q[j + 4] & 63;+ } else {+ *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);+ *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);+ }+}++//========================- 2-bit (de)-quantization++void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ uint8_t L[QK_K];+ uint8_t Laux[16];+ float weights[16];+ float mins[QK_K/16];+ float scales[QK_K/16];++ const float q4scale = 15.f;++ for (int i = 0; i < nb; i++) {+ float max_scale = 0; // as we are deducting the min, scales are always positive+ float max_min = 0;+ for (int j = 0; j < QK_K/16; ++j) {+ for (int l = 0; l < 16; ++l) weights[l] = fabsf(x[16*j + l]);+ scales[j] = make_qkx2_quants(16, 3, x + 16*j, weights, L + 16*j, &mins[j], Laux, -0.5f, 0.1f, 15, true);+ float scale = scales[j];+ if (scale > max_scale) {+ max_scale = scale;+ }+ float min = mins[j];+ if (min > max_min) {+ max_min = min;+ }+ }++ if (max_scale > 0) {+ float iscale = q4scale/max_scale;+ for (int j = 0; j < QK_K/16; ++j) {+ int l = nearest_int(iscale*scales[j]);+ y[i].scales[j] = l;+ }+ y[i].d = GGML_FP32_TO_FP16(max_scale/q4scale);+ } else {+ for (int j = 0; j < QK_K/16; ++j) y[i].scales[j] = 0;+ y[i].d = GGML_FP32_TO_FP16(0.f);+ }+ if (max_min > 0) {+ float iscale = q4scale/max_min;+ for (int j = 0; j < QK_K/16; ++j) {+ int l = nearest_int(iscale*mins[j]);+ y[i].scales[j] |= (l << 4);+ }+ y[i].dmin = GGML_FP32_TO_FP16(max_min/q4scale);+ } else {+ y[i].dmin = GGML_FP32_TO_FP16(0.f);+ }+ for (int j = 0; j < QK_K/16; ++j) {+ const float d = GGML_FP16_TO_FP32(y[i].d) * (y[i].scales[j] & 0xF);+ if (!d) continue;+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * (y[i].scales[j] >> 4);+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int((x[16*j + ii] + dm)/d);+ l = MAX(0, MIN(3, l));+ L[16*j + ii] = l;+ }+ }++ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);+ }+ }++ x += QK_K;+ }+}++void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const float min = GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * q = x[i].qs;++ int is = 0;+ float dl, ml;+ for (int n = 0; n < QK_K; n += 128) {+ int shift = 0;+ for (int j = 0; j < 4; ++j) {++ uint8_t sc = x[i].scales[is++];+ dl = d * (sc & 0xF); ml = min * (sc >> 4);+ for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml;++ sc = x[i].scales[is++];+ dl = d * (sc & 0xF); ml = min * (sc >> 4);+ for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml;++ shift += 2;+ }+ q += 32;+ }+ }+}++static float make_qkx3_quants(int n, int nmax, const float * GGML_RESTRICT x, const float * GGML_RESTRICT weights,+ uint8_t * GGML_RESTRICT L, float * GGML_RESTRICT the_min, uint8_t * GGML_RESTRICT Laux,+ float rmin, float rdelta, int nstep, bool use_mad) {+ float min = x[0];+ float max = x[0];+ float sum_w = weights ? weights[0] : x[0]*x[0];+ float sum_x = sum_w * x[0];+#ifdef HAVE_BUGGY_APPLE_LINKER+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7+ for (volatile int i = 1; i < n; ++i) {+#else+ for (int i = 1; i < n; ++i) {+#endif+ if (x[i] < min) min = x[i];+ if (x[i] > max) max = x[i];+ float w = weights ? weights[i] : x[i]*x[i];+ sum_w += w;+ sum_x += w * x[i];+ }+ if (min > 0) {+ min = 0;+ }+ if (max <= min) {+ memset(L, 0, n);+ *the_min = -min;+ return 0.f;+ }+ float iscale = nmax/(max - min);+ float scale = 1/iscale;+ float best_mad = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale*(x[i] - min));+ L[i] = MAX(0, MIN(nmax, l));+ float diff = scale * L[i] + min - x[i];+ diff = use_mad ? fabsf(diff) : diff*diff;+ float w = weights ? weights[i] : x[i]*x[i];+ best_mad += w * diff;+ }+ if (nstep < 1) {+ *the_min = -min;+ return scale;+ }+ for (int is = 0; is <= nstep; ++is) {+ iscale = (rmin + rdelta*is + nmax)/(max - min);+ float sum_l = 0, sum_l2 = 0, sum_xl = 0;+ for (int i = 0; i < n; ++i) {+ // min is the best fit so far and can be at or near max, so v can be inf, nan or out of range for nearest_int+ const float v = iscale*(x[i] - min);+ const int l = v > 0 ? nearest_int(MIN(v, nmax)) : 0;+ Laux[i] = l;+ float w = weights ? weights[i] : x[i]*x[i];+ sum_l += w*l;+ sum_l2 += w*l*l;+ sum_xl += w*l*x[i];+ }+ float D = sum_w * sum_l2 - sum_l * sum_l;+ if (D > 0) {+ float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D;+ float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D;+ if (this_min > 0) {+ this_min = 0;+ this_scale = sum_xl / sum_l2;+ }+ float mad = 0;+ for (int i = 0; i < n; ++i) {+ float diff = this_scale * Laux[i] + this_min - x[i];+ diff = use_mad ? fabsf(diff) : diff*diff;+ float w = weights ? weights[i] : x[i]*x[i];+ mad += w * diff;+ }+ if (mad < best_mad) {+ for (int i = 0; i < n; ++i) {+ L[i] = Laux[i];+ }+ best_mad = mad;+ scale = this_scale;+ min = this_min;+ }+ }+ }+ *the_min = -min;+ return scale;+}++static float make_qp_quants(int n, int nmax, const float * GGML_RESTRICT x, uint8_t * GGML_RESTRICT L, const float * quant_weights) {+ float max = 0;+ for (int i = 0; i < n; ++i) {+ max = MAX(max, x[i]);+ }+ if (max < GROUP_MAX_EPS) { // all zero+ for (int i = 0; i < n; ++i) { L[i] = 0; }+ return 0.f;+ }+ float iscale = nmax / max;+ for (int i = 0; i < n; ++i) {+ L[i] = nearest_int(iscale * x[i]);+ }+ float scale = 1/iscale;+ float best_mse = 0;+ for (int i = 0; i < n; ++i) {+ float diff = x[i] - scale*L[i];+ float w = quant_weights[i];+ best_mse += w*diff*diff;+ }+ for (int is = -4; is <= 4; ++is) {+ if (is == 0) continue;+ float iscale_is = (0.1f*is + nmax)/max;+ float scale_is = 1/iscale_is;+ float mse = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale_is*x[i]);+ l = MIN(nmax, l);+ float diff = x[i] - scale_is*l;+ float w = quant_weights[i];+ mse += w*diff*diff;+ }+ if (mse < best_mse) {+ best_mse = mse;+ iscale = iscale_is;+ }+ }+ float sumlx = 0;+ float suml2 = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale * x[i]);+ l = MIN(nmax, l);+ L[i] = l;+ float w = quant_weights[i];+ sumlx += w*x[i]*l;+ suml2 += w*l*l;+ }+ for (int itry = 0; itry < 5; ++itry) {+ int n_changed = 0;+ for (int i = 0; i < n; ++i) {+ float w = quant_weights[i];+ float slx = sumlx - w*x[i]*L[i];+ float sl2 = suml2 - w*L[i]*L[i];+ if (slx > 0 && sl2 > 0) {+ int new_l = nearest_int(x[i] * sl2 / slx);+ new_l = MIN(nmax, new_l);+ if (new_l != L[i]) {+ slx += w*x[i]*new_l;+ sl2 += w*new_l*new_l;+ if (slx*slx*suml2 > sumlx*sumlx*sl2) {+ L[i] = new_l; sumlx = slx; suml2 = sl2;+ ++n_changed;+ }+ }+ }+ }+ if (!n_changed) {+ break;+ }+ }+ return suml2 > 0.0f ? sumlx / suml2 : 0.0f;+}++static void quantize_row_q2_K_impl(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int k, const float * GGML_RESTRICT quant_weights) {+ GGML_ASSERT(quant_weights);+ assert(k % QK_K == 0);+ const int nb = k / QK_K;+ const bool requantize = true;++ uint8_t L[QK_K];+ uint8_t Laux[16];+ float mins[QK_K/16];+ float scales[QK_K/16];+ float sw[QK_K/16];+ float weight[16];+ uint8_t Ls[QK_K/16], Lm[QK_K/16];++ for (int i = 0; i < nb; i++) {+ memset(sw, 0, QK_K/16*sizeof(float));+ float sumx2 = 0;+ for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];+ float sigma2 = sumx2/QK_K;+ for (int j = 0; j < QK_K/16; ++j) {+ const float * GGML_RESTRICT qw = quant_weights + QK_K * i + 16*j;+ for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]);+ for (int l = 0; l < QK_K/16; ++l) sw[j] += weight[l];+ scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);+ }++ float dm, mm;+ dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw);+ mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw);++ y[i].d = GGML_FP32_TO_FP16(dm);+ y[i].dmin = GGML_FP32_TO_FP16(mm);+ dm = GGML_FP16_TO_FP32(y[i].d);+ mm = GGML_FP16_TO_FP32(y[i].dmin);++ for (int j = 0; j < QK_K/16; ++j) {+ y[i].scales[j] = Ls[j] | (Lm[j] << 4);+ }++ if (requantize) {+ for (int j = 0; j < QK_K/16; ++j) {+ const float d = dm * (y[i].scales[j] & 0xF);+ if (!d) continue;+ const float m = mm * (y[i].scales[j] >> 4);+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int((x[16*j + ii] + m)/d);+ l = MAX(0, MIN(3, l));+ L[16*j + ii] = l;+ }+ }+ }++ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);+ }+ }++ x += QK_K;+ }+}++size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ size_t row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q2_K_ref(src, dst, (int64_t)nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q2_K_impl(src, (block_q2_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}++//========================= 3-bit (de)-quantization++void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ int8_t L[QK_K];+ float scales[QK_K / 16];++ for (int i = 0; i < nb; i++) {++ float max_scale = 0;+ float amax = 0;+ for (int j = 0; j < QK_K/16; ++j) {+ scales[j] = make_q3_quants(16, 4, x + 16*j, L + 16*j, true);+ float scale = fabsf(scales[j]);+ if (scale > amax) {+ amax = scale; max_scale = scales[j];+ }+ }++ memset(y[i].scales, 0, 12);+ if (max_scale) {+ float iscale = -32.f/max_scale;+ for (int j = 0; j < QK_K/16; ++j) {+ int8_t l = nearest_int(iscale*scales[j]);+ l = MAX(-32, MIN(31, l)) + 32;+ if (j < 8) {+ y[i].scales[j] = l & 0xF;+ } else {+ y[i].scales[j-8] |= ((l & 0xF) << 4);+ }+ l >>= 4;+ y[i].scales[j%4 + 8] |= (l << (2*(j/4)));+ }+ y[i].d = GGML_FP32_TO_FP16(1/iscale);+ } else {+ y[i].d = GGML_FP32_TO_FP16(0.f);+ }++ int8_t sc;+ for (int j = 0; j < QK_K/16; ++j) {+ sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4;+ sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32;+ float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) {+ continue;+ }+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int(x[16*j + ii]/d);+ l = MAX(-4, MIN(3, l));+ L[16*j + ii] = l + 4;+ }+ }++ memset(y[i].hmask, 0, QK_K/8);+ // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc.+ int m = 0;+ uint8_t hm = 1;+ for (int j = 0; j < QK_K; ++j) {+ if (L[j] > 3) {+ y[i].hmask[m] |= hm;+ L[j] -= 4;+ }+ if (++m == QK_K/8) {+ m = 0; hm <<= 1;+ }+ }+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);+ }+ }++ x += QK_K;+ }+}++void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ const uint32_t kmask1 = 0x03030303;+ const uint32_t kmask2 = 0x0f0f0f0f;++ uint32_t aux[4];+ const int8_t * scales = (const int8_t*)aux;++ for (int i = 0; i < nb; i++) {++ const float d_all = GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * GGML_RESTRICT q = x[i].qs;+ const uint8_t * GGML_RESTRICT hm = x[i].hmask;+ uint8_t m = 1;++ memcpy(aux, x[i].scales, 12);+ uint32_t tmp = aux[2];+ aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);+ aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);+ aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);+ aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);++ int is = 0;+ float dl;+ for (int n = 0; n < QK_K; n += 128) {+ int shift = 0;+ for (int j = 0; j < 4; ++j) {++ dl = d_all * (scales[is++] - 32);+ for (int l = 0; l < 16; ++l) {+ *y++ = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4));+ }++ dl = d_all * (scales[is++] - 32);+ for (int l = 0; l < 16; ++l) {+ *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4));+ }++ shift += 2;+ m <<= 1;+ }+ q += 32;+ }++ }+}++static void quantize_row_q3_K_impl(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t n_per_row, const float * GGML_RESTRICT quant_weights) {+ assert(n_per_row % QK_K == 0);+ const int nb = n_per_row / QK_K;++ int8_t L[QK_K];+ float scales[QK_K / 16];+ float weight[16];+ float sw[QK_K / 16];+ int8_t Ls[QK_K / 16];++ for (int i = 0; i < nb; i++) {++ float sumx2 = 0;+ for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];+ float sigma2 = 2*sumx2/QK_K;++ for (int j = 0; j < QK_K/16; ++j) {+ if (quant_weights) {+ const float * qw = quant_weights + QK_K * i + 16*j;+ for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]);+ } else {+ for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l];+ }+ float sumw = 0;+ for (int l = 0; l < 16; ++l) sumw += weight[l];+ sw[j] = sumw;++ scales[j] = make_qx_quants(16, 4, x + 16*j, L + 16*j, 1, weight);++ }++ memset(y[i].scales, 0, 12);++ float d_block = make_qx_quants(QK_K/16, 32, scales, Ls, 1, sw);+ for (int j = 0; j < QK_K/16; ++j) {+ int l = Ls[j];+ if (j < 8) {+ y[i].scales[j] = l & 0xF;+ } else {+ y[i].scales[j-8] |= ((l & 0xF) << 4);+ }+ l >>= 4;+ y[i].scales[j%4 + 8] |= (l << (2*(j/4)));+ }+ y[i].d = GGML_FP32_TO_FP16(d_block);++ int8_t sc;+ for (int j = 0; j < QK_K/16; ++j) {+ sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4;+ sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32;+ float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) {+ continue;+ }+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int(x[16*j + ii]/d);+ l = MAX(-4, MIN(3, l));+ L[16*j + ii] = l + 4;+ }+ }++ memset(y[i].hmask, 0, QK_K/8);+ // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc.+ int m = 0;+ uint8_t hm = 1;+ for (int j = 0; j < QK_K; ++j) {+ if (L[j] > 3) {+ y[i].hmask[m] |= hm;+ L[j] -= 4;+ }+ if (++m == QK_K/8) {+ m = 0; hm <<= 1;+ }+ }+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);+ }+ }++ x += QK_K;+ }+}++size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ size_t row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q3_K_ref(src, dst, (int64_t)nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q3_K_impl(src, (block_q3_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}++// ====================== 4-bit (de)-quantization++void quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ uint8_t L[QK_K];+ uint8_t Laux[32];+ float weights[32];+ float mins[QK_K/32];+ float scales[QK_K/32];++ for (int i = 0; i < nb; i++) {+ float max_scale = 0; // as we are deducting the min, scales are always positive+ float max_min = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ //scales[j] = make_qkx1_quants(32, 15, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);+ float sum_x2 = 0;+ for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];+ float av_x = sqrtf(sum_x2/32);+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);+ scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false);+ float scale = scales[j];+ if (scale > max_scale) {+ max_scale = scale;+ }+ float min = mins[j];+ if (min > max_min) {+ max_min = min;+ }+ }++ float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;+ float inv_min = max_min > 0 ? 63.f/max_min : 0.f;+ for (int j = 0; j < QK_K/32; ++j) {+ uint8_t ls = nearest_int(inv_scale*scales[j]);+ uint8_t lm = nearest_int(inv_min*mins[j]);+ ls = MIN(63, ls);+ lm = MIN(63, lm);+ if (j < 4) {+ y[i].scales[j] = ls;+ y[i].scales[j+4] = lm;+ } else {+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);+ y[i].scales[j-4] |= ((ls >> 4) << 6);+ y[i].scales[j-0] |= ((lm >> 4) << 6);+ }+ }+ y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);+ y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);++ uint8_t sc, m;+ for (int j = 0; j < QK_K/32; ++j) {+ get_scale_min_k4(j, y[i].scales, &sc, &m);+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) continue;+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;+ for (int ii = 0; ii < 32; ++ii) {+ int l = nearest_int((x[32*j + ii] + dm)/d);+ l = MAX(0, MIN(15, l));+ L[32*j + ii] = l;+ }+ }++ uint8_t * q = y[i].qs;+ for (int j = 0; j < QK_K; j += 64) {+ for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4);+ q += 32;+ }++ x += QK_K;+ }+}++void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ for (int i = 0; i < nb; i++) {+ const uint8_t * q = x[i].qs;++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const float min = GGML_FP16_TO_FP32(x[i].dmin);++ int is = 0;+ uint8_t sc, m;+ for (int j = 0; j < QK_K; j += 64) {+ get_scale_min_k4(is + 0, x[i].scales, &sc, &m);+ const float d1 = d * sc; const float m1 = min * m;+ get_scale_min_k4(is + 1, x[i].scales, &sc, &m);+ const float d2 = d * sc; const float m2 = min * m;+ for (int l = 0; l < 32; ++l) *y++ = d1 * (q[l] & 0xF) - m1;+ for (int l = 0; l < 32; ++l) *y++ = d2 * (q[l] >> 4) - m2;+ q += 32; is += 2;+ }+ }+}++static void quantize_row_q4_K_impl(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {+ assert(n_per_row % QK_K == 0);+ const int64_t nb = n_per_row / QK_K;++ uint8_t L[QK_K];+ uint8_t Laux[32];+ uint8_t Ls[QK_K/32];+ uint8_t Lm[QK_K/32];+ float weights[32];+ float sw[QK_K/32];+ float mins[QK_K/32];+ float scales[QK_K/32];++ for (int i = 0; i < nb; i++) {++ float sum_x2 = 0;+ for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];+ float sigma2 = 2*sum_x2/QK_K;+ float av_x = sqrtf(sigma2);++ for (int j = 0; j < QK_K/32; ++j) {+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*i + 32*j;+ for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);+ } else {+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);+ }+ float sumw = 0;+ for (int l = 0; l < 32; ++l) sumw += weights[l];+ sw[j] = sumw;+ scales[j] = make_qkx3_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);+ }++ float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw);+ float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw);+ for (int j = 0; j < QK_K/32; ++j) {+ uint8_t ls = Ls[j];+ uint8_t lm = Lm[j];+ if (j < 4) {+ y[i].scales[j] = ls;+ y[i].scales[j+4] = lm;+ } else {+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);+ y[i].scales[j-4] |= ((ls >> 4) << 6);+ y[i].scales[j-0] |= ((lm >> 4) << 6);+ }+ }+ y[i].d = GGML_FP32_TO_FP16(d_block);+ y[i].dmin = GGML_FP32_TO_FP16(m_block);++ uint8_t sc, m;+ for (int j = 0; j < QK_K/32; ++j) {+ get_scale_min_k4(j, y[i].scales, &sc, &m);+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) continue;+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;+ for (int ii = 0; ii < 32; ++ii) {+ int l = nearest_int((x[32*j + ii] + dm)/d);+ l = MAX(0, MIN(15, l));+ L[32*j + ii] = l;+ }+ }+ uint8_t * q = y[i].qs;+ for (int j = 0; j < QK_K; j += 64) {+ for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4);+ q += 32;+ }++ x += QK_K;++ }+}++size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q4_K_ref(src, dst, (int64_t)nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q4_K_impl(src, (block_q4_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}++// ====================== 5-bit (de)-quantization++void quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ uint8_t L[QK_K];+ float mins[QK_K/32];+ float scales[QK_K/32];+ float weights[32];+ uint8_t Laux[32];++ for (int i = 0; i < nb; i++) {+ float max_scale = 0; // as we are deducting the min, scales are always positive+ float max_min = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ //scales[j] = make_qkx1_quants(32, 31, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);+ float sum_x2 = 0;+ for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];+ float av_x = sqrtf(sum_x2/32);+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);+ scales[j] = make_qkx2_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.5f, 0.1f, 15, false);+ float scale = scales[j];+ if (scale > max_scale) {+ max_scale = scale;+ }+ float min = mins[j];+ if (min > max_min) {+ max_min = min;+ }+ }++ float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;+ float inv_min = max_min > 0 ? 63.f/max_min : 0.f;+ for (int j = 0; j < QK_K/32; ++j) {+ uint8_t ls = nearest_int(inv_scale*scales[j]);+ uint8_t lm = nearest_int(inv_min*mins[j]);+ ls = MIN(63, ls);+ lm = MIN(63, lm);+ if (j < 4) {+ y[i].scales[j] = ls;+ y[i].scales[j+4] = lm;+ } else {+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);+ y[i].scales[j-4] |= ((ls >> 4) << 6);+ y[i].scales[j-0] |= ((lm >> 4) << 6);+ }+ }+ y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);+ y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);++ uint8_t sc, m;+ for (int j = 0; j < QK_K/32; ++j) {+ get_scale_min_k4(j, y[i].scales, &sc, &m);+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) continue;+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;+ for (int ii = 0; ii < 32; ++ii) {+ int l = nearest_int((x[32*j + ii] + dm)/d);+ l = MAX(0, MIN(31, l));+ L[32*j + ii] = l;+ }+ }++ uint8_t * GGML_RESTRICT qh = y[i].qh;+ uint8_t * GGML_RESTRICT ql = y[i].qs;+ memset(qh, 0, QK_K/8);++ uint8_t m1 = 1, m2 = 2;+ for (int n = 0; n < QK_K; n += 64) {+ for (int j = 0; j < 32; ++j) {+ int l1 = L[n + j];+ if (l1 > 15) {+ l1 -= 16; qh[j] |= m1;+ }+ int l2 = L[n + j + 32];+ if (l2 > 15) {+ l2 -= 16; qh[j] |= m2;+ }+ ql[j] = l1 | (l2 << 4);+ }+ m1 <<= 2; m2 <<= 2;+ ql += 32;+ }++ x += QK_K;+ }+}++void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int i = 0; i < nb; i++) {+ const uint8_t * ql = x[i].qs;+ const uint8_t * qh = x[i].qh;++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const float min = GGML_FP16_TO_FP32(x[i].dmin);++ int is = 0;+ uint8_t sc, m;+ uint8_t u1 = 1, u2 = 2;+ for (int j = 0; j < QK_K; j += 64) {+ get_scale_min_k4(is + 0, x[i].scales, &sc, &m);+ const float d1 = d * sc; const float m1 = min * m;+ get_scale_min_k4(is + 1, x[i].scales, &sc, &m);+ const float d2 = d * sc; const float m2 = min * m;+ for (int l = 0; l < 32; ++l) *y++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1;+ for (int l = 0; l < 32; ++l) *y++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2;+ ql += 32; is += 2;+ u1 <<= 2; u2 <<= 2;+ }+ }+}++static void quantize_row_q5_K_impl(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {+ assert(n_per_row % QK_K == 0);+ const int64_t nb = n_per_row / QK_K;++ uint8_t L[QK_K];+ uint8_t Laux[32];+ uint8_t Ls[QK_K/32];+ uint8_t Lm[QK_K/32];+ float mins[QK_K/32];+ float scales[QK_K/32];+ float sw[QK_K/32];+ float weights[32];++ for (int i = 0; i < nb; i++) {++ float sum_x2 = 0;+ for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];+ float sigma2 = 2*sum_x2/QK_K;+ float av_x = sqrtf(sigma2);++ for (int j = 0; j < QK_K/32; ++j) {+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*i + 32*j;+ for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);+ } else {+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);+ }+ float sumw = 0;+ for (int l = 0; l < 32; ++l) sumw += weights[l];+ sw[j] = sumw;++ scales[j] = make_qkx3_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);+ }++ float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw);+ float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw);++ for (int j = 0; j < QK_K/32; ++j) {+ uint8_t ls = Ls[j];+ uint8_t lm = Lm[j];+ ls = MIN(63, ls);+ lm = MIN(63, lm);+ if (j < 4) {+ y[i].scales[j] = ls;+ y[i].scales[j+4] = lm;+ } else {+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);+ y[i].scales[j-4] |= ((ls >> 4) << 6);+ y[i].scales[j-0] |= ((lm >> 4) << 6);+ }+ }+ y[i].d = GGML_FP32_TO_FP16(d_block);+ y[i].dmin = GGML_FP32_TO_FP16(m_block);++ uint8_t sc, m;+ for (int j = 0; j < QK_K/32; ++j) {+ get_scale_min_k4(j, y[i].scales, &sc, &m);+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) continue;+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;+ for (int ii = 0; ii < 32; ++ii) {+ int l = nearest_int((x[32*j + ii] + dm)/d);+ l = MAX(0, MIN(31, l));+ L[32*j + ii] = l;+ }+ }++ uint8_t * GGML_RESTRICT qh = y[i].qh;+ uint8_t * GGML_RESTRICT ql = y[i].qs;+ memset(qh, 0, QK_K/8);++ uint8_t m1 = 1, m2 = 2;+ for (int n = 0; n < QK_K; n += 64) {+ for (int j = 0; j < 32; ++j) {+ int l1 = L[n + j];+ if (l1 > 15) {+ l1 -= 16; qh[j] |= m1;+ }+ int l2 = L[n + j + 32];+ if (l2 > 15) {+ l2 -= 16; qh[j] |= m2;+ }+ ql[j] = l1 | (l2 << 4);+ }+ m1 <<= 2; m2 <<= 2;+ ql += 32;+ }++ x += QK_K;++ }+}++size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q5_K_ref(src, dst, (int64_t)nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q5_K_impl(src, (block_q5_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}++// ====================== 6-bit (de)-quantization++void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ int8_t L[QK_K];+ float scales[QK_K/16];++ for (int i = 0; i < nb; i++) {++ float max_scale = 0;+ float max_abs_scale = 0;++ for (int ib = 0; ib < QK_K/16; ++ib) {++ const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);+ scales[ib] = scale;++ const float abs_scale = fabsf(scale);+ if (abs_scale > max_abs_scale) {+ max_abs_scale = abs_scale;+ max_scale = scale;+ }++ }++ if (max_abs_scale < GROUP_MAX_EPS) {+ memset(&y[i], 0, sizeof(block_q6_K));+ y[i].d = GGML_FP32_TO_FP16(0.f);+ x += QK_K;+ continue;+ }++ float iscale = -128.f/max_scale;+ y[i].d = GGML_FP32_TO_FP16(1/iscale);+ for (int ib = 0; ib < QK_K/16; ++ib) {+ y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));+ }++ for (int j = 0; j < QK_K/16; ++j) {+ float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j];+ if (!d) {+ continue;+ }+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int(x[16*j + ii]/d);+ l = MAX(-32, MIN(31, l));+ L[16*j + ii] = l + 32;+ }+ }++ uint8_t * GGML_RESTRICT ql = y[i].ql;+ uint8_t * GGML_RESTRICT qh = y[i].qh;+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ const uint8_t q1 = L[j + l + 0] & 0xF;+ const uint8_t q2 = L[j + l + 32] & 0xF;+ const uint8_t q3 = L[j + l + 64] & 0xF;+ const uint8_t q4 = L[j + l + 96] & 0xF;+ ql[l+ 0] = q1 | (q3 << 4);+ ql[l+32] = q2 | (q4 << 4);+ qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6);+ }+ ql += 64;+ qh += 32;+ }++ x += QK_K;+ }+}++void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * GGML_RESTRICT ql = x[i].ql;+ const uint8_t * GGML_RESTRICT qh = x[i].qh;+ const int8_t * GGML_RESTRICT sc = x[i].scales;++ for (int n = 0; n < QK_K; n += 128) {+ for (int l = 0; l < 32; ++l) {+ int is = l/16;+ const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;+ const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;+ const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;+ const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;+ y[l + 0] = d * sc[is + 0] * q1;+ y[l + 32] = d * sc[is + 2] * q2;+ y[l + 64] = d * sc[is + 4] * q3;+ y[l + 96] = d * sc[is + 6] * q4;+ }+ y += 128;+ ql += 64;+ qh += 32;+ sc += 8;+ }+ }+}++static void quantize_row_q6_K_impl(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {+ assert(n_per_row % QK_K == 0);+ const int64_t nb = n_per_row / QK_K;++ int8_t L[QK_K];+ float scales[QK_K/16];+ //float weights[16];++ for (int i = 0; i < nb; i++) {++ //float sum_x2 = 0;+ //for (int j = 0; j < QK_K; ++j) sum_x2 += x[j]*x[j];+ //float sigma2 = sum_x2/QK_K;++ float max_scale = 0;+ float max_abs_scale = 0;++ for (int ib = 0; ib < QK_K/16; ++ib) {++ float scale;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*i + 16*ib;+ //for (int j = 0; j < 16; ++j) weights[j] = qw[j] * sqrtf(sigma2 + x[16*ib + j]*x[16*ib + j]);+ //scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, weights);+ scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, qw);+ } else {+ scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);+ }+ scales[ib] = scale;++ const float abs_scale = fabsf(scale);+ if (abs_scale > max_abs_scale) {+ max_abs_scale = abs_scale;+ max_scale = scale;+ }++ }++ if (max_abs_scale < GROUP_MAX_EPS) {+ memset(&y[i], 0, sizeof(block_q6_K));+ y[i].d = GGML_FP32_TO_FP16(0.f);+ x += QK_K;+ continue;+ }++ float iscale = -128.f/max_scale;+ y[i].d = GGML_FP32_TO_FP16(1/iscale);+ for (int ib = 0; ib < QK_K/16; ++ib) {+ y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));+ }++ for (int j = 0; j < QK_K/16; ++j) {+ float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j];+ if (!d) {+ continue;+ }+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int(x[16*j + ii]/d);+ l = MAX(-32, MIN(31, l));+ L[16*j + ii] = l + 32;+ }+ }++ uint8_t * GGML_RESTRICT ql = y[i].ql;+ uint8_t * GGML_RESTRICT qh = y[i].qh;+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ const uint8_t q1 = L[j + l + 0] & 0xF;+ const uint8_t q2 = L[j + l + 32] & 0xF;+ const uint8_t q3 = L[j + l + 64] & 0xF;+ const uint8_t q4 = L[j + l + 96] & 0xF;+ ql[l+ 0] = q1 | (q3 << 4);+ ql[l+32] = q2 | (q4 << 4);+ qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6);+ }+ ql += 64;+ qh += 32;+ }++ x += QK_K;++ }+}++size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ size_t row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q6_K_ref(src, dst, (int64_t)nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q6_K_impl(src, (block_q6_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}++static void quantize_row_q4_0_impl(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {+ static_assert(QK4_0 == 32, "QK4_0 must be 32");++ if (!quant_weights) {+ quantize_row_q4_0_ref(x, y, n_per_row);+ return;+ }++ float weight[QK4_0];+ int8_t L[QK4_0];++ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int64_t nb = n_per_row/QK4_0;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK4_0 * ib;+ const float * qw = quant_weights + QK4_0 * ib;+ for (int j = 0; j < QK4_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float d = make_qx_quants(QK4_0, 8, xb, L, 1, weight);+ y[ib].d = GGML_FP32_TO_FP16(d);+ for (int j = 0; j < 16; ++j) {+ y[ib].qs[j] = L[j] | (L[j+16] << 4);+ }+ }+}++size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ if (!quant_weights) {+ quantize_row_q1_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q1_0, n_per_row);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q1_0, n_per_row);+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q1_0_ref(src, (block_q1_0*)qrow, n_per_row);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ if (!quant_weights) {+ quantize_row_q2_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q2_0, n_per_row);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q2_0, n_per_row);+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q2_0_ref(src, (block_q2_0*)qrow, n_per_row);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ if (!quant_weights) {+ quantize_row_q4_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q4_0, n_per_row);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_0, n_per_row);+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q4_0_impl(src, (block_q4_0*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++static void quantize_row_q4_1_impl(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {+ static_assert(QK4_1 == 32, "QK4_1 must be 32");++ if (!quant_weights) {+ quantize_row_q4_1_ref(x, y, n_per_row);+ return;+ }++ float weight[QK4_1];+ uint8_t L[QK4_1], Laux[QK4_1];++ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int64_t nb = n_per_row/QK4_1;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK4_1 * ib;+ const float * qw = quant_weights + QK4_1 * ib;+ for (int j = 0; j < QK4_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float min;+ float d = make_qkx3_quants(QK4_1, 15, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);+ y[ib].d = GGML_FP32_TO_FP16(d);+ y[ib].m = GGML_FP32_TO_FP16(-min);+ for (int j = 0; j < 16; ++j) {+ y[ib].qs[j] = L[j] | (L[j+16] << 4);+ }+ }+}++size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ if (!quant_weights) {+ quantize_row_q4_1_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q4_1, n_per_row);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_1, n_per_row);+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q4_1_impl(src, (block_q4_1*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++static void quantize_row_q5_0_impl(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {+ static_assert(QK5_0 == 32, "QK5_0 must be 32");++ if (!quant_weights) {+ quantize_row_q5_0_ref(x, y, n_per_row);+ return;+ }++ float weight[QK5_0];+ int8_t L[QK5_0];++ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int64_t nb = n_per_row/QK5_0;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK5_0 * ib;+ const float * qw = quant_weights + QK5_0 * ib;+ for (int j = 0; j < QK5_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float d = make_qx_quants(QK5_0, 16, xb, L, 1, weight);+ y[ib].d = GGML_FP32_TO_FP16(d);++ uint32_t qh = 0;++ for (int j = 0; j < 16; ++j) {+ const uint8_t xi0 = L[j];+ const uint8_t xi1 = L[j+16];+ y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);++ // get the 5-th bit and store it in qh at the right position+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);+ }++ memcpy(&y[ib].qh, &qh, sizeof(qh));+ }+}++size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ if (!quant_weights) {+ quantize_row_q5_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q5_0, n_per_row);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_0, n_per_row);+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q5_0_impl(src, (block_q5_0*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++static void quantize_row_q5_1_impl(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {+ static_assert(QK5_1 == 32, "QK5_1 must be 32");++ if (!quant_weights) {+ quantize_row_q5_1_ref(x, y, n_per_row);+ return;+ }++ float weight[QK5_1];+ uint8_t L[QK5_1], Laux[QK5_1];++ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int64_t nb = n_per_row/QK5_1;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK5_1 * ib;+ const float * qw = quant_weights + QK5_1 * ib;+ for (int j = 0; j < QK5_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float min;+ float d = make_qkx3_quants(QK5_1, 31, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);+ y[ib].d = GGML_FP32_TO_FP16(d);+ y[ib].m = GGML_FP32_TO_FP16(-min);++ uint32_t qh = 0;+ for (int j = 0; j < 16; ++j) {+ const uint8_t xi0 = L[j];+ const uint8_t xi1 = L[j+16];+ y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);+ // get the 5-th bit and store it in qh at the right position+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);+ }+ memcpy(&y[ib].qh, &qh, sizeof(qh));+ }+}++size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ if (!quant_weights) {+ quantize_row_q5_1_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q5_1, n_per_row);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_1, n_per_row);+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q5_1_impl(src, (block_q5_1*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ (void)quant_weights; // not used+ const size_t row_size = ggml_row_size(GGML_TYPE_Q8_0, n_per_row);+ quantize_row_q8_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * row_size;+}++size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_UNUSED(quant_weights);+ quantize_row_mxfp4_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_MXFP4, n_per_row);+}++size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_UNUSED(quant_weights);+ quantize_row_nvfp4_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_NVFP4, n_per_row);+}++// ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs)++void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int64_t i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK_K; j++) {+ const float v = x[j];+ amax = MAX(amax, fabsf(v));+ }++ const float d = amax;+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ // 5 elements per byte, along 32 bytes+ for (size_t j = 0; j < sizeof(y->qs) - sizeof(y->qs) % 32; j += 32) {+ for (size_t m = 0; m < 32; ++m) {+ uint8_t q = 0;+ for (size_t n = 0; n < 5; ++n) {+ int xi = lroundf(x[m + n*32] * id) + 1; // -1, 0, 1 -> 0, 1, 2+ q *= 3;+ q += xi;+ }+ // ceiling division (243 == pow(3, 5))+ q = ((uint16_t)q * 256 + (243 - 1)) / 243;+ y[i].qs[j + m] = q;+ }+ x += 5*32;+ }+ // along 16 bytes+ for (size_t j = sizeof(y->qs) - sizeof(y->qs) % 32; j < sizeof(y->qs); j += 16) {+ for (size_t m = 0; m < 16; ++m) {+ uint8_t q = 0;+ for (size_t n = 0; n < 5; ++n) {+ int xi = lroundf(x[m + n*16] * id) + 1; // -1, 0, 1 -> 0, 1, 2+ q *= 3;+ q += xi;+ }+ // ceiling division (243 == pow(3, 5))+ q = ((uint16_t)q * 256 + (243 - 1)) / 243;+ y[i].qs[j + m] = q;+ }+ x += 5*16;+ }+ // 4 elements per byte+ for (size_t j = 0; j < sizeof(y->qh); ++j) {+ uint8_t q = 0;+ for (size_t m = 0; m < 4; ++m) {+ // -1, 0, 1 -> 0, 1, 2+ int xi = lroundf(x[j + m*sizeof(y->qh)] * id) + 1;+ q *= 3;+ q += xi;+ }+ // shift the first value to the most significant trit+ q *= 3;+ // ceiling division (243 == pow(3, 5))+ q = ((uint16_t)q * 256 + (243 - 1)) / 243;+ y[i].qh[j] = q;+ }+ x += 4*sizeof(y->qh);+ }+}++void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int64_t i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK_K; j++) {+ const float v = x[j];+ amax = MAX(amax, fabsf(v));+ }++ const float d = amax;+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ for (size_t j = 0; j < sizeof(y->qs); j += 32) {+ for (size_t m = 0; m < 32; ++m) {+ uint8_t q = 0;+ for (size_t n = 0; n < 4; ++n) {+ // -1, 0, 1 -> 0, 1, 2+ int xi = lroundf(x[m + n*32] * id) + 1;+ q += (xi & 3) << (2*n);+ }+ y[i].qs[j + m] = q;+ }+ x += 4*32;+ }+ }+}++size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ (void)quant_weights; // not used+ const size_t row_size = ggml_row_size(GGML_TYPE_TQ1_0, n_per_row);+ quantize_row_tq1_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * row_size;+}++size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ (void)quant_weights; // not used+ const size_t row_size = ggml_row_size(GGML_TYPE_TQ2_0, n_per_row);+ quantize_row_tq2_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * row_size;+}++void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243};++ for (int64_t i = 0; i < nb; ++i) {++ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (size_t j = 0; j < sizeof(x->qs) - sizeof(x->qs) % 32; j += 32) {+ for (size_t n = 0; n < 5; ++n) {+ for (size_t m = 0; m < 32; ++m) {+ uint8_t q = x[i].qs[j + m] * pow3[n];+ int16_t xi = ((uint16_t) q * 3) >> 8;+ *y++ = (float) (xi - 1) * d;+ }+ }+ }+ for (size_t j = sizeof(x->qs) - sizeof(x->qs) % 32; j < sizeof(x->qs); j += 16) {+ for (size_t n = 0; n < 5; ++n) {+ for (size_t m = 0; m < 16; ++m) {+ uint8_t q = x[i].qs[j + m] * pow3[n];+ int16_t xi = ((uint16_t) q * 3) >> 8;+ *y++ = (float) (xi - 1) * d;+ }+ }+ }++ for (size_t n = 0; n < 4; ++n) {+ for (size_t j = 0; j < sizeof(x->qh); ++j) {+ uint8_t q = x[i].qh[j] * pow3[n];+ int16_t xi = ((uint16_t) q * 3) >> 8;+ *y++ = (float) (xi - 1) * d;+ }+ }+ }+}++void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int64_t i = 0; i < nb; ++i) {++ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (size_t j = 0; j < sizeof(x->qs); j += 32) {+ for (size_t l = 0; l < 4; ++l) {+ for (size_t m = 0; m < 32; ++m) {+ int8_t q = (x[i].qs[j + m] >> (l*2)) & 3;+ *y++ = (float) (q - 1) * d;+ }+ }+ }+ }+}++// ====================== "True" 2-bit (de)-quantization++void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ uint32_t aux32[2];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ memcpy(aux32, x[i].qs + 4*ib32, 2*sizeof(uint32_t));+ const float db = d * (0.5f + (aux32[1] >> 28)) * 0.25f;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);+ const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];+ for (int j = 0; j < 8; ++j) {+ y[j] = db * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+ }+ y += 8;+ }+ }+ }+}++// ====================== 2.3125 bpw (de)-quantization++void dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ float db[2];++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f;+ db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (x[i].qs[4*ib32 + l] & 511));+ const uint8_t signs = ksigns_iq2xs[x[i].qs[4*ib32 + l] >> 9];+ for (int j = 0; j < 8; ++j) {+ y[j] = db[l/2] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+ }+ y += 8;+ }+ }+ }+}++// ====================== 2.5625 bpw (de)-quantization++void dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ float db[2];++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const uint8_t * qs = x[i].qs;+ const uint8_t * qh = x[i].qh;+ const uint8_t * signs = qs + QK_K/8;++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f;+ db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f;+ for (int l = 0; l < 4; ++l) {+ const float dl = db[l/2];+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));+ for (int j = 0; j < 8; ++j) {+ y[j] = dl * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1.f : 1.f);+ }+ y += 8;+ }+ qs += 4;+ signs += 4;+ }+ }+}++// ====================== 3.0625 bpw (de)-quantization++void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ uint32_t aux32;++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const uint8_t * qs = x[i].qs;+ const uint8_t * scales_and_signs = qs + QK_K/4;++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ memcpy(&aux32, scales_and_signs + 4*ib32, sizeof(uint32_t));+ const float db = d * (0.5f + (aux32 >> 28)) * 0.5f;+ for (int l = 0; l < 4; ++l) {+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + qs[2*l+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + qs[2*l+1]);+ for (int j = 0; j < 4; ++j) {+ y[j+0] = db * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = db * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+ y += 8;+ }+ qs += 8;+ }+ }+}++// ====================== 3.3125 bpw (de)-quantization++void dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const uint8_t * qs = x[i].qs;+ const uint8_t * qh = x[i].qh;+ const uint8_t * signs = x[i].signs;++ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const float db1 = d * (1 + 2*(x[i].scales[ib32/2] & 0xf));+ const float db2 = d * (1 + 2*(x[i].scales[ib32/2] >> 4));+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256)));+ for (int j = 0; j < 4; ++j) {+ y[j+0] = db1 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = db1 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+ y += 8;+ }+ qs += 8;+ signs += 4;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256)));+ for (int j = 0; j < 4; ++j) {+ y[j+0] = db2 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = db2 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+ y += 8;+ }+ qh += 2;+ qs += 8;+ signs += 4;+ }+ }+}++// ====================== 1.5625 bpw (de)-quantization++void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;++ for (int ib = 0; ib < QK_K/32; ++ib) {+ const float dl = d * (2*((qh[ib] >> 12) & 7) + 1);+ const float delta = qh[ib] & 0x8000 ? -IQ1S_DELTA : IQ1S_DELTA;+ for (int l = 0; l < 4; ++l) {+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));+ for (int j = 0; j < 8; ++j) {+ y[j] = dl * (grid[j] + delta);+ }+ y += 8;+ }+ qs += 4;+ }+ }+}++void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ float delta[4];+ uint16_t idx[4];++ iq1m_scale_t scale;++ for (int i = 0; i < nb; i++) {++ const uint16_t * sc = (const uint16_t *)x[i].scales;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);+ const float d = GGML_FP16_TO_FP32(scale.f16);++ const uint8_t * qs = x[i].qs;+ const uint8_t * qh = x[i].qh;++ for (int ib = 0; ib < QK_K/32; ++ib) {+ const float dl1 = d * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1);+ const float dl2 = d * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1);++ idx[0] = qs[0] | ((qh[0] << 8) & 0x700);+ idx[1] = qs[1] | ((qh[0] << 4) & 0x700);+ idx[2] = qs[2] | ((qh[1] << 8) & 0x700);+ idx[3] = qs[3] | ((qh[1] << 4) & 0x700);+ delta[0] = qh[0] & 0x08 ? -IQ1S_DELTA : IQ1S_DELTA;+ delta[1] = qh[0] & 0x80 ? -IQ1S_DELTA : IQ1S_DELTA;+ delta[2] = qh[1] & 0x08 ? -IQ1S_DELTA : IQ1S_DELTA;+ delta[3] = qh[1] & 0x80 ? -IQ1S_DELTA : IQ1S_DELTA;+ for (int l = 0; l < 2; ++l) {+ const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);+ for (int j = 0; j < 8; ++j) {+ y[j] = dl1 * (grid[j] + delta[l]);+ }+ y += 8;+ }+ for (int l = 2; l < 4; ++l) {+ const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);+ for (int j = 0; j < 8; ++j) {+ y[j] = dl2 * (grid[j] + delta[l]);+ }+ y += 8;+ }+ qs += 4;+ qh += 2;+ }+ }+}++void dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK4_NL == 0);+ const int64_t nb = k / QK4_NL;++ for (int i = 0; i < nb; i++) {++ const uint8_t * qs = x[i].qs;++ const float d = GGML_FP16_TO_FP32(x[i].d);+ for (int j = 0; j < QK4_NL/2; ++j) {+ y[j+ 0] = d * kvalues_iq4nl[qs[j] & 0xf];+ y[j+QK4_NL/2] = d * kvalues_iq4nl[qs[j] >> 4];+ }+ y += QK4_NL;+ qs += QK4_NL/2;+ }+}++void dequantize_row_iq4_xs(const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int i = 0; i < nb; i++) {++ const uint8_t * qs = x[i].qs;++ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int ib = 0; ib < QK_K/32; ++ib) {+ const int ls = ((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4);+ const float dl = d * (ls - 32);+ for (int j = 0; j < 16; ++j) {+ y[j+ 0] = dl * kvalues_iq4nl[qs[j] & 0xf];+ y[j+16] = dl * kvalues_iq4nl[qs[j] >> 4];+ }+ y += 32;+ qs += 16;+ }+ }+}++//===================================== Q8_K ==============================================++void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int i = 0; i < nb; i++) {++ float max = 0;+ float amax = 0;+ for (int j = 0; j < QK_K; ++j) {+ float ax = fabsf(x[j]);+ if (ax > amax) {+ amax = ax; max = x[j];+ }+ }+ if (!amax) {+ y[i].d = 0;+ memset(y[i].qs, 0, QK_K);+ x += QK_K;+ continue;+ }+ //const float iscale = -128.f/max;+ // We need this change for IQ2_XXS, else the AVX implementation becomes very awkward+ const float iscale = -127.f/max;+ for (int j = 0; j < QK_K; ++j) {+ int v = nearest_int(iscale*x[j]);+ y[i].qs[j] = MIN(127, v);+ }+ for (int j = 0; j < QK_K/16; ++j) {+ int sum = 0;+ for (int ii = 0; ii < 16; ++ii) {+ sum += y[i].qs[j*16 + ii];+ }+ y[i].bsums[j] = sum;+ }+ y[i].d = 1/iscale;+ x += QK_K;+ }+}++void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ const int64_t nb = k / QK_K;++ for (int i = 0; i < nb; i++) {+ for (int j = 0; j < QK_K; ++j) {+ *y++ = x[i].d * x[i].qs[j];+ }+ }+}++// ================================ IQ2 quantization =============================================++typedef struct {+ uint64_t * grid;+ int * map;+ uint16_t * neighbours;+} iq2_entry_t;++static iq2_entry_t iq2_data[4] = {+ {NULL, NULL, NULL},+ {NULL, NULL, NULL},+ {NULL, NULL, NULL},+ {NULL, NULL, NULL},+};++static inline int iq2_data_index(enum ggml_type type) {+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);+ return type == GGML_TYPE_IQ2_XXS ? 0 :+ type == GGML_TYPE_IQ2_XS ? 1 :+ type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 2 : 3;+}++static inline int iq2_grid_size(enum ggml_type type) {+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);+ return type == GGML_TYPE_IQ2_XXS ? 256 :+ type == GGML_TYPE_IQ2_XS ? 512 :+ type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? NGRID_IQ1S : 1024;+}++static int iq2_compare_func(const void * left, const void * right) {+ const int * l = (const int *)left;+ const int * r = (const int *)right;+ return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;+}++void iq2xs_init_impl(enum ggml_type type) {+ const int gindex = iq2_data_index(type);+ const int grid_size = iq2_grid_size(type);+ if (iq2_data[gindex].grid) {+ return;+ }+ static const uint16_t kgrid_2bit_256[256] = {+ 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97,+ 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642,+ 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288,+ 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113,+ 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240,+ 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400,+ 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260,+ 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872,+ 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516,+ 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561,+ 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488,+ 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545,+ 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874,+ 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856,+ 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142,+ 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268,+ };+ static const uint16_t kgrid_2bit_512[512] = {+ 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,+ 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257,+ 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340,+ 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597,+ 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096,+ 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348,+ 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065,+ 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441,+ 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160,+ 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372,+ 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125,+ 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652,+ 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197,+ 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549,+ 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894,+ 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388,+ 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480,+ 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773,+ 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473,+ 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436,+ 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497,+ 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162,+ 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528,+ 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745,+ 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234,+ 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025,+ 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810,+ 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984,+ 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462,+ 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960,+ 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048,+ 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690,+ };+ static const uint16_t kgrid_1bit_2048[NGRID_IQ1S] = {+ 0, 2, 5, 8, 10, 17, 21, 32, 34, 40, 42, 69, 81, 84, 86, 101,+ 128, 130, 136, 138, 149, 160, 162, 168, 170, 260, 261, 273, 276, 278, 281, 282,+ 293, 321, 326, 329, 338, 341, 346, 353, 356, 358, 360, 389, 401, 404, 406, 421,+ 512, 514, 520, 522, 533, 544, 546, 552, 554, 581, 593, 601, 612, 617, 640, 642,+ 648, 650, 657, 661, 665, 672, 674, 680, 682, 1041, 1044, 1046, 1061, 1089, 1097, 1109,+ 1114, 1124, 1125, 1169, 1177, 1189, 1281, 1284, 1285, 1286, 1301, 1304, 1306, 1321, 1344, 1349,+ 1354, 1360, 1361, 1364, 1365, 1366, 1369, 1376, 1378, 1381, 1384, 1386, 1409, 1425, 1429, 1432,+ 1434, 1441, 1444, 1445, 1446, 1449, 1556, 1561, 1601, 1604, 1616, 1618, 1621, 1624, 1632, 1633,+ 1638, 1641, 1669, 1681, 1684, 1689, 2048, 2050, 2056, 2058, 2069, 2080, 2082, 2088, 2090, 2117,+ 2129, 2134, 2149, 2176, 2178, 2184, 2186, 2197, 2208, 2210, 2216, 2218, 2309, 2321, 2324, 2329,+ 2340, 2341, 2369, 2384, 2385, 2389, 2401, 2404, 2409, 2449, 2452, 2454, 2457, 2469, 2560, 2562,+ 2568, 2570, 2581, 2592, 2594, 2600, 2602, 2629, 2641, 2649, 2657, 2661, 2688, 2690, 2693, 2696,+ 2698, 2709, 2720, 2722, 2728, 2730, 4112, 4113, 4116, 4121, 4132, 4133, 4161, 4164, 4176, 4181,+ 4184, 4193, 4196, 4197, 4201, 4241, 4244, 4246, 4257, 4261, 4353, 4356, 4358, 4361, 4368, 4370,+ 4373, 4376, 4385, 4388, 4393, 4421, 4426, 4432, 4433, 4434, 4436, 4437, 4438, 4441, 4448, 4453,+ 4484, 4498, 4501, 4513, 4516, 4625, 4628, 4630, 4645, 4672, 4678, 4681, 4690, 4693, 4696, 4698,+ 4708, 4710, 4741, 4753, 4756, 4758, 4773, 5121, 5126, 5129, 5140, 5141, 5144, 5145, 5153, 5158,+ 5185, 5189, 5190, 5192, 5194, 5201, 5204, 5205, 5206, 5209, 5218, 5221, 5224, 5252, 5257, 5264,+ 5268, 5269, 5272, 5273, 5274, 5281, 5284, 5285, 5289, 5378, 5381, 5386, 5393, 5396, 5397, 5398,+ 5401, 5408, 5410, 5413, 5416, 5418, 5441, 5444, 5445, 5446, 5457, 5458, 5460, 5461, 5462, 5465,+ 5466, 5473, 5476, 5477, 5478, 5481, 5504, 5506, 5508, 5509, 5512, 5514, 5520, 5521, 5524, 5525,+ 5526, 5529, 5530, 5536, 5538, 5541, 5633, 5636, 5637, 5638, 5653, 5654, 5656, 5658, 5665, 5670,+ 5696, 5698, 5700, 5701, 5704, 5706, 5713, 5717, 5718, 5720, 5721, 5729, 5732, 5733, 5736, 5737,+ 5738, 5766, 5770, 5778, 5781, 5796, 5801, 6161, 6166, 6181, 6209, 6212, 6214, 6217, 6224, 6229,+ 6232, 6234, 6240, 6241, 6244, 6246, 6249, 6277, 6289, 6292, 6309, 6416, 6418, 6421, 6426, 6433,+ 6437, 6466, 6468, 6469, 6472, 6481, 6484, 6485, 6486, 6489, 6490, 6496, 6501, 6506, 6537, 6545,+ 6546, 6549, 6552, 6561, 6566, 6569, 6665, 6678, 6692, 6694, 6724, 6726, 6729, 6736, 6738, 6741,+ 6744, 6753, 6758, 6761, 6789, 6801, 6806, 6810, 8192, 8194, 8200, 8202, 8213, 8224, 8226, 8229,+ 8232, 8234, 8261, 8273, 8281, 8289, 8293, 8320, 8322, 8328, 8330, 8341, 8352, 8354, 8357, 8360,+ 8362, 8453, 8465, 8468, 8473, 8485, 8514, 8516, 8521, 8533, 8536, 8538, 8545, 8548, 8549, 8550,+ 8581, 8592, 8598, 8601, 8613, 8705, 8712, 8714, 8721, 8725, 8736, 8738, 8744, 8746, 8773, 8785,+ 8790, 8793, 8805, 8833, 8840, 8842, 8849, 8853, 8864, 8866, 8872, 8874, 9221, 9236, 9238, 9241,+ 9253, 9284, 9285, 9286, 9289, 9298, 9301, 9304, 9306, 9318, 9349, 9361, 9364, 9369, 9377, 9381,+ 9481, 9493, 9505, 9513, 9536, 9541, 9544, 9553, 9556, 9557, 9561, 9570, 9573, 9576, 9609, 9616,+ 9620, 9621, 9624, 9626, 9633, 9636, 9638, 9641, 9733, 9744, 9746, 9753, 9765, 9793, 9801, 9813,+ 9824, 9825, 9833, 9860, 9862, 9872, 9882, 10240, 10242, 10248, 10250, 10261, 10272, 10274, 10280, 10282,+ 10309, 10321, 10324, 10341, 10368, 10370, 10376, 10378, 10400, 10402, 10408, 10410, 10505, 10513, 10516, 10521,+ 10533, 10566, 10569, 10578, 10581, 10593, 10596, 10598, 10601, 10629, 10640, 10646, 10649, 10660, 10661, 10752,+ 10754, 10760, 10762, 10784, 10786, 10792, 10794, 10821, 10833, 10838, 10841, 10853, 10880, 10882, 10888, 10890,+ 10901, 10912, 10914, 10920, 10922, 16389, 16401, 16406, 16421, 16457, 16466, 16469, 16472, 16474, 16481, 16484,+ 16486, 16532, 16537, 16545, 16550, 16640, 16641, 16644, 16646, 16649, 16658, 16661, 16662, 16664, 16666, 16673,+ 16678, 16681, 16709, 16712, 16714, 16721, 16724, 16725, 16726, 16729, 16730, 16741, 16744, 16746, 16769, 16772,+ 16774, 16784, 16786, 16789, 16800, 16801, 16802, 16901, 16913, 16916, 16918, 16933, 16961, 16978, 16981, 16986,+ 16996, 17001, 17033, 17044, 17061, 17409, 17429, 17433, 17449, 17477, 17480, 17482, 17489, 17492, 17493, 17494,+ 17505, 17506, 17509, 17512, 17514, 17537, 17542, 17545, 17552, 17554, 17557, 17568, 17569, 17577, 17665, 17666,+ 17669, 17674, 17681, 17684, 17685, 17686, 17689, 17696, 17701, 17706, 17729, 17732, 17733, 17734, 17737, 17744,+ 17745, 17748, 17749, 17750, 17752, 17753, 17761, 17764, 17765, 17766, 17769, 17794, 17796, 17797, 17800, 17809,+ 17812, 17813, 17814, 17817, 17818, 17829, 17832, 17834, 17921, 17925, 17929, 17940, 17941, 17944, 17946, 17953,+ 17956, 17961, 17984, 17986, 17989, 17992, 18000, 18001, 18002, 18005, 18006, 18009, 18018, 18021, 18024, 18049,+ 18053, 18058, 18068, 18069, 18081, 18084, 18086, 18437, 18449, 18453, 18458, 18469, 18498, 18505, 18512, 18517,+ 18520, 18529, 18532, 18534, 18537, 18565, 18577, 18580, 18582, 18585, 18597, 18689, 18693, 18694, 18698, 18704,+ 18708, 18709, 18712, 18721, 18724, 18726, 18752, 18757, 18762, 18769, 18770, 18772, 18773, 18774, 18777, 18784,+ 18786, 18789, 18790, 18794, 18822, 18825, 18834, 18837, 18838, 18840, 18849, 18852, 18854, 18857, 18966, 19012,+ 19014, 19017, 19029, 19032, 19034, 19044, 19049, 19092, 19109, 20481, 20484, 20485, 20486, 20489, 20498, 20501,+ 20506, 20513, 20516, 20521, 20544, 20549, 20552, 20561, 20564, 20565, 20566, 20569, 20581, 20584, 20614, 20617,+ 20629, 20632, 20640, 20641, 20646, 20649, 20741, 20744, 20745, 20746, 20753, 20756, 20757, 20758, 20760, 20761,+ 20768, 20773, 20774, 20776, 20778, 20801, 20804, 20805, 20806, 20809, 20816, 20817, 20818, 20820, 20821, 20822,+ 20824, 20825, 20826, 20833, 20836, 20837, 20838, 20841, 20866, 20869, 20881, 20884, 20885, 20886, 20889, 20896,+ 20901, 20906, 20993, 20998, 21010, 21013, 21018, 21025, 21028, 21058, 21061, 21066, 21073, 21076, 21077, 21078,+ 21081, 21090, 21093, 21125, 21136, 21138, 21141, 21145, 21146, 21156, 21508, 21509, 21521, 21524, 21525, 21526,+ 21528, 21529, 21537, 21541, 21544, 21546, 21569, 21572, 21573, 21574, 21577, 21578, 21584, 21585, 21588, 21589,+ 21590, 21592, 21593, 21594, 21601, 21602, 21604, 21605, 21606, 21609, 21632, 21640, 21642, 21649, 21652, 21653,+ 21654, 21657, 21665, 21668, 21669, 21674, 21761, 21762, 21764, 21765, 21766, 21769, 21776, 21777, 21778, 21780,+ 21781, 21782, 21785, 21786, 21793, 21796, 21797, 21798, 21801, 21824, 21825, 21826, 21828, 21829, 21830, 21832,+ 21833, 21840, 21841, 21842, 21844, 21845, 21846, 21848, 21849, 21850, 21856, 21857, 21860, 21861, 21862, 21864,+ 21865, 21866, 21889, 21892, 21893, 21897, 21898, 21904, 21905, 21908, 21909, 21910, 21912, 21913, 21921, 21924,+ 21925, 21926, 21929, 22016, 22017, 22018, 22020, 22022, 22024, 22025, 22033, 22036, 22037, 22040, 22041, 22048,+ 22049, 22050, 22052, 22053, 22054, 22056, 22057, 22081, 22085, 22086, 22088, 22089, 22090, 22096, 22097, 22098,+ 22100, 22101, 22102, 22104, 22105, 22106, 22113, 22116, 22117, 22121, 22146, 22149, 22150, 22152, 22153, 22154,+ 22161, 22165, 22170, 22178, 22181, 22182, 22184, 22185, 22532, 22533, 22534, 22537, 22544, 22549, 22552, 22561,+ 22570, 22597, 22600, 22602, 22609, 22612, 22613, 22614, 22616, 22617, 22624, 22626, 22628, 22629, 22658, 22665,+ 22672, 22674, 22677, 22680, 22689, 22697, 22785, 22786, 22789, 22794, 22801, 22804, 22805, 22806, 22809, 22821,+ 22849, 22852, 22853, 22854, 22857, 22864, 22865, 22866, 22868, 22869, 22870, 22872, 22873, 22874, 22881, 22884,+ 22885, 22886, 22889, 22913, 22917, 22921, 22929, 22932, 22933, 22934, 22936, 22937, 22949, 23044, 23048, 23061,+ 23066, 23072, 23077, 23078, 23081, 23109, 23112, 23113, 23121, 23125, 23126, 23128, 23129, 23138, 23141, 23144,+ 23146, 23169, 23178, 23186, 23189, 23190, 23192, 23194, 23201, 24581, 24596, 24598, 24601, 24613, 24644, 24656,+ 24661, 24662, 24664, 24666, 24673, 24676, 24678, 24681, 24705, 24726, 24741, 24833, 24836, 24838, 24841, 24850,+ 24853, 24865, 24866, 24870, 24873, 24901, 24905, 24913, 24917, 24918, 24921, 24933, 24934, 24938, 24964, 24970,+ 24978, 24981, 24993, 24998, 25001, 25105, 25110, 25113, 25152, 25153, 25158, 25173, 25174, 25176, 25184, 25221,+ 25233, 25238, 25253, 25617, 25618, 25621, 25622, 25626, 25633, 25638, 25641, 25664, 25666, 25669, 25672, 25674,+ 25681, 25684, 25685, 25686, 25689, 25690, 25696, 25698, 25701, 25732, 25733, 25737, 25744, 25746, 25748, 25749,+ 25750, 25752, 25754, 25761, 25764, 25769, 25861, 25864, 25866, 25873, 25877, 25878, 25881, 25924, 25925, 25926,+ 25929, 25936, 25937, 25940, 25941, 25942, 25945, 25953, 25956, 25957, 25958, 25961, 25990, 25993, 25994, 26001,+ 26005, 26006, 26009, 26010, 26018, 26021, 26022, 26024, 26114, 26121, 26133, 26144, 26150, 26152, 26153, 26176,+ 26181, 26184, 26186, 26193, 26196, 26197, 26198, 26200, 26202, 26208, 26213, 26216, 26240, 26242, 26245, 26250,+ 26260, 26262, 26264, 26265, 26272, 26276, 26278, 26282, 26646, 26649, 26661, 26689, 26706, 26709, 26714, 26721,+ 26729, 26757, 26769, 26776, 26790, 26881, 26884, 26896, 26901, 26913, 26916, 26918, 26921, 26944, 26945, 26949,+ 26950, 26952, 26961, 26964, 26965, 26966, 26969, 26976, 26981, 26986, 27010, 27012, 27018, 27029, 27041, 27044,+ 27045, 27049, 27153, 27158, 27160, 27201, 27204, 27209, 27216, 27221, 27224, 27226, 27236, 27237, 27241, 27270,+ 27284, 27288, 27290, 27302, 32768, 32770, 32776, 32778, 32800, 32802, 32808, 32810, 32837, 32848, 32849, 32852,+ 32854, 32857, 32869, 32896, 32898, 32904, 32906, 32917, 32928, 32930, 32936, 32938, 33029, 33041, 33044, 33046,+ 33049, 33061, 33089, 33092, 33097, 33104, 33106, 33109, 33110, 33112, 33113, 33124, 33126, 33129, 33157, 33161,+ 33172, 33174, 33177, 33189, 33280, 33282, 33288, 33290, 33301, 33312, 33314, 33320, 33322, 33361, 33364, 33369,+ 33381, 33408, 33410, 33416, 33418, 33429, 33440, 33442, 33448, 33450, 33812, 33817, 33857, 33860, 33873, 33877,+ 33882, 33889, 33892, 33897, 33940, 33945, 34049, 34057, 34066, 34069, 34074, 34086, 34089, 34112, 34113, 34117,+ 34120, 34129, 34132, 34133, 34134, 34137, 34138, 34149, 34150, 34152, 34154, 34177, 34180, 34182, 34185, 34192,+ 34194, 34197, 34200, 34214, 34321, 34326, 34329, 34341, 34369, 34372, 34377, 34378, 34384, 34389, 34393, 34394,+ 34401, 34406, 34410, 34437, 34449, 34458, 34468, 34816, 34818, 34824, 34826, 34837, 34848, 34850, 34856, 34858,+ 34881, 34885, 34897, 34900, 34905, 34917, 34921, 34944, 34946, 34952, 34954, 34965, 34976, 34978, 34984, 34986,+ 35077, 35078, 35089, 35092, 35094, 35109, 35137, 35140, 35142, 35145, 35152, 35154, 35157, 35162, 35169, 35172,+ 35205, 35222, 35225, 35237, 35328, 35330, 35336, 35338, 35349, 35360, 35362, 35368, 35370, 35397, 35409, 35412,+ 35414, 35456, 35458, 35464, 35466, 35477, 35488, 35490, 35496, 35498, 36869, 36881, 36886, 36888, 36889, 36901,+ 36929, 36934, 36937, 36949, 36952, 36954, 36969, 36970, 36997, 37009, 37012, 37014, 37017, 37029, 37121, 37124,+ 37126, 37129, 37136, 37141, 37144, 37146, 37153, 37156, 37158, 37161, 37184, 37189, 37200, 37201, 37204, 37205,+ 37206, 37209, 37218, 37221, 37252, 37254, 37266, 37269, 37272, 37281, 37284, 37286, 37289, 37381, 37393, 37396,+ 37401, 37413, 37444, 37446, 37449, 37456, 37458, 37461, 37464, 37478, 37481, 37509, 37524, 37526, 37545, 37889,+ 37892, 37894, 37904, 37909, 37912, 37926, 37952, 37962, 37969, 37972, 37973, 37974, 37976, 37977, 37984, 37985,+ 37986, 37989, 38020, 38022, 38034, 38036, 38037, 38040, 38049, 38057, 38144, 38149, 38152, 38154, 38160, 38161,+ 38164, 38165, 38166, 38169, 38177, 38181, 38185, 38186, 38209, 38212, 38213, 38214, 38217, 38224, 38225, 38226,+ 38228, 38229, 38230, 38232, 38233, 38234, 38241, 38244, 38245, 38246, 38249, 38273, 38277, 38280, 38289, 38290,+ 38292, 38293, 38294, 38297, 38298, 38304, 38306, 38309, 38312, 38314, 38401, 38404, 38416, 38421, 38425, 38432,+ 38438, 38441, 38469, 38472, 38473, 38481, 38482, 38485, 38486, 38489, 38501, 38504, 38530, 38532, 38537, 38538,+ 38546, 38548, 38549, 38564, 38566, 38569, 38917, 38934, 38937, 38949, 38977, 38982, 38992, 38994, 38997, 38998,+ 39002, 39012, 39013, 39045, 39057, 39062, 39065, 39077, 39172, 39174, 39177, 39184, 39186, 39189, 39192, 39194,+ 39200, 39201, 39204, 39206, 39232, 39234, 39237, 39240, 39242, 39249, 39252, 39253, 39254, 39257, 39266, 39269,+ 39270, 39274, 39297, 39300, 39312, 39314, 39317, 39322, 39329, 39334, 39429, 39445, 39461, 39492, 39494, 39497,+ 39504, 39509, 39512, 39521, 39557, 39569, 39572, 39573, 39574, 40960, 40962, 40968, 40970, 40981, 40992, 40994,+ 41000, 41002, 41029, 41041, 41044, 41046, 41049, 41088, 41090, 41096, 41098, 41109, 41120, 41122, 41128, 41130,+ 41221, 41225, 41233, 41236, 41238, 41241, 41242, 41286, 41289, 41297, 41301, 41304, 41306, 41313, 41316, 41349,+ 41360, 41362, 41366, 41369, 41474, 41480, 41482, 41488, 41497, 41506, 41512, 41514, 41541, 41553, 41558, 41561,+ 41573, 41600, 41602, 41608, 41610, 41621, 41632, 41634, 41640, 41642, 42009, 42021, 42049, 42052, 42064, 42068,+ 42069, 42072, 42074, 42081, 42085, 42086, 42088, 42089, 42117, 42246, 42249, 42256, 42258, 42261, 42264, 42278,+ 42281, 42306, 42309, 42321, 42324, 42325, 42326, 42329, 42341, 42346, 42369, 42372, 42373, 42374, 42377, 42386,+ 42389, 42392, 42501, 42513, 42518, 42522, 42529, 42533, 42564, 42566, 42570, 42578, 42581, 42582, 42584, 42592,+ 42594, 42630, 42640, 42645, 42646, 42649, 42657, 42660, 42662, 43008, 43010, 43016, 43018, 43040, 43042, 43048,+ 43050, 43089, 43092, 43094, 43097, 43136, 43138, 43144, 43146, 43157, 43168, 43170, 43176, 43178, 43269, 43284,+ 43289, 43297, 43301, 43329, 43344, 43349, 43354, 43361, 43366, 43369, 43408, 43414, 43520, 43522, 43528, 43530,+ 43552, 43554, 43560, 43562, 43601, 43604, 43606, 43648, 43650, 43656, 43658, 43669, 43680, 43682, 43688, 43690,+ };+ static const uint16_t kgrid_2bit_1024[1024] = {+ 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,+ 73, 80, 82, 85, 88, 97, 100, 102, 105, 128, 130, 133, 136, 145, 148, 160,+ 165, 170, 257, 260, 262, 265, 272, 274, 277, 280, 289, 292, 320, 322, 325, 328,+ 337, 340, 342, 345, 352, 357, 360, 385, 388, 400, 402, 405, 417, 420, 512, 514,+ 517, 520, 529, 532, 544, 554, 577, 580, 582, 585, 592, 597, 640, 645, 650, 660,+ 674, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1062, 1065, 1088, 1090, 1093,+ 1096, 1098, 1105, 1108, 1110, 1113, 1120, 1122, 1125, 1153, 1156, 1158, 1161, 1168, 1173, 1176,+ 1185, 1188, 1280, 1282, 1285, 1288, 1290, 1297, 1300, 1302, 1305, 1312, 1317, 1320, 1345, 1348,+ 1350, 1353, 1360, 1362, 1365, 1368, 1377, 1380, 1408, 1410, 1413, 1416, 1425, 1428, 1440, 1537,+ 1540, 1542, 1545, 1552, 1557, 1600, 1605, 1608, 1617, 1620, 1632, 1665, 1668, 1680, 2048, 2050,+ 2053, 2056, 2065, 2068, 2070, 2073, 2080, 2085, 2090, 2113, 2116, 2118, 2121, 2128, 2130, 2133,+ 2136, 2145, 2148, 2176, 2181, 2196, 2218, 2305, 2308, 2320, 2322, 2325, 2328, 2337, 2368, 2373,+ 2376, 2385, 2388, 2400, 2433, 2448, 2560, 2577, 2580, 2594, 2600, 2602, 2640, 2713, 4097, 4100,+ 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4134, 4160, 4162, 4165, 4168, 4177, 4180, 4182,+ 4185, 4192, 4194, 4197, 4200, 4225, 4228, 4230, 4240, 4245, 4248, 4257, 4260, 4352, 4354, 4357,+ 4360, 4362, 4369, 4372, 4374, 4377, 4384, 4386, 4389, 4392, 4417, 4420, 4422, 4425, 4432, 4434,+ 4437, 4440, 4449, 4452, 4480, 4482, 4485, 4488, 4497, 4500, 4609, 4612, 4617, 4624, 4629, 4641,+ 4644, 4672, 4677, 4689, 4692, 4737, 4740, 4752, 5120, 5122, 5125, 5128, 5137, 5140, 5142, 5145,+ 5152, 5157, 5160, 5185, 5188, 5190, 5193, 5200, 5202, 5205, 5208, 5217, 5220, 5248, 5250, 5253,+ 5256, 5265, 5268, 5280, 5377, 5380, 5382, 5385, 5392, 5394, 5397, 5400, 5409, 5412, 5440, 5442,+ 5445, 5448, 5457, 5460, 5472, 5505, 5508, 5520, 5632, 5637, 5640, 5649, 5652, 5664, 5697, 5700,+ 5712, 5760, 5802, 6145, 6148, 6150, 6153, 6160, 6165, 6168, 6177, 6208, 6210, 6213, 6216, 6225,+ 6228, 6240, 6273, 6276, 6400, 6402, 6405, 6408, 6417, 6420, 6432, 6465, 6468, 6480, 6505, 6562,+ 6660, 6672, 6720, 6742, 8192, 8194, 8197, 8200, 8209, 8212, 8214, 8217, 8224, 8229, 8234, 8257,+ 8260, 8272, 8274, 8277, 8292, 8320, 8330, 8340, 8362, 8449, 8452, 8464, 8466, 8469, 8481, 8512,+ 8514, 8517, 8529, 8532, 8544, 8577, 8580, 8592, 8704, 8714, 8738, 8744, 8746, 8772, 8784, 8840,+ 8842, 8872, 9217, 9220, 9222, 9225, 9232, 9237, 9240, 9249, 9252, 9280, 9282, 9285, 9288, 9297,+ 9300, 9312, 9345, 9348, 9360, 9472, 9477, 9480, 9489, 9492, 9504, 9537, 9540, 9552, 9574, 9600,+ 9729, 9732, 9744, 9792, 9817, 10240, 10245, 10257, 10260, 10305, 10308, 10320, 10378, 10410, 10497, 10500,+ 10512, 10645, 10762, 10786, 10852, 10888, 10890, 16385, 16388, 16390, 16393, 16400, 16402, 16405, 16408, 16410,+ 16417, 16420, 16422, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16470, 16473, 16480, 16482, 16485, 16513,+ 16516, 16528, 16533, 16536, 16545, 16548, 16640, 16642, 16645, 16648, 16657, 16660, 16662, 16665, 16672, 16674,+ 16677, 16705, 16708, 16710, 16713, 16720, 16722, 16725, 16728, 16737, 16740, 16768, 16770, 16773, 16776, 16785,+ 16788, 16800, 16897, 16900, 16912, 16914, 16917, 16920, 16932, 16960, 16965, 16968, 16977, 16980, 16992, 17025,+ 17028, 17408, 17410, 17413, 17416, 17418, 17425, 17428, 17430, 17433, 17440, 17442, 17445, 17448, 17473, 17476,+ 17478, 17481, 17488, 17490, 17493, 17496, 17505, 17508, 17536, 17538, 17541, 17544, 17553, 17556, 17568, 17665,+ 17668, 17670, 17673, 17680, 17682, 17685, 17688, 17697, 17700, 17728, 17730, 17733, 17736, 17745, 17748, 17760,+ 17770, 17793, 17796, 17808, 17920, 17922, 17925, 17928, 17937, 17940, 17952, 17985, 17988, 18000, 18048, 18085,+ 18433, 18436, 18441, 18448, 18450, 18453, 18456, 18465, 18468, 18496, 18498, 18501, 18504, 18513, 18516, 18528,+ 18564, 18576, 18688, 18690, 18693, 18696, 18705, 18708, 18720, 18753, 18756, 18768, 18816, 18838, 18945, 18948,+ 18960, 19008, 20480, 20482, 20485, 20488, 20497, 20500, 20502, 20505, 20512, 20514, 20517, 20520, 20545, 20548,+ 20550, 20553, 20560, 20562, 20565, 20568, 20577, 20580, 20608, 20610, 20613, 20616, 20625, 20628, 20737, 20740,+ 20742, 20745, 20752, 20754, 20757, 20760, 20769, 20772, 20800, 20802, 20805, 20808, 20817, 20820, 20832, 20865,+ 20868, 20880, 20992, 20997, 21000, 21009, 21012, 21024, 21057, 21060, 21072, 21097, 21120, 21505, 21508, 21510,+ 21513, 21520, 21522, 21525, 21528, 21537, 21540, 21568, 21570, 21573, 21576, 21585, 21588, 21600, 21633, 21636,+ 21648, 21760, 21762, 21765, 21768, 21777, 21780, 21792, 21825, 21828, 21840, 21888, 22017, 22020, 22032, 22054,+ 22080, 22528, 22530, 22533, 22536, 22545, 22548, 22560, 22593, 22596, 22608, 22618, 22656, 22785, 22788, 22800,+ 22848, 23040, 23065, 23173, 23208, 24577, 24580, 24582, 24592, 24594, 24597, 24600, 24609, 24612, 24640, 24645,+ 24648, 24657, 24660, 24672, 24708, 24720, 24832, 24834, 24837, 24840, 24849, 24852, 24864, 24897, 24900, 24912,+ 24960, 24985, 25092, 25104, 25152, 25174, 25249, 25600, 25605, 25608, 25617, 25620, 25632, 25665, 25668, 25680,+ 25728, 25857, 25860, 25872, 25920, 25930, 25960, 26002, 26112, 26260, 26625, 26628, 26640, 26725, 26776, 26880,+ 26922, 27202, 27297, 32768, 32770, 32773, 32776, 32785, 32788, 32793, 32800, 32805, 32833, 32836, 32848, 32850,+ 32853, 32856, 32865, 32896, 32901, 32913, 32916, 33025, 33028, 33033, 33040, 33042, 33045, 33048, 33057, 33060,+ 33088, 33090, 33093, 33096, 33105, 33108, 33153, 33156, 33168, 33193, 33280, 33285, 33290, 33297, 33300, 33345,+ 33348, 33360, 33793, 33796, 33798, 33801, 33808, 33810, 33813, 33816, 33825, 33856, 33858, 33861, 33864, 33873,+ 33876, 33888, 33921, 33924, 33936, 34048, 34050, 34053, 34056, 34065, 34068, 34080, 34113, 34116, 34128, 34176,+ 34186, 34305, 34308, 34320, 34345, 34368, 34816, 34821, 34833, 34836, 34881, 34884, 34896, 34978, 35073, 35076,+ 35136, 35173, 35362, 35416, 35418, 35458, 35490, 36865, 36868, 36873, 36880, 36882, 36885, 36888, 36900, 36928,+ 36930, 36933, 36936, 36945, 36948, 36960, 36993, 36996, 37008, 37120, 37125, 37137, 37140, 37185, 37188, 37200,+ 37210, 37377, 37380, 37392, 37440, 37542, 37888, 37890, 37893, 37896, 37905, 37908, 37920, 37953, 37956, 37968,+ 38016, 38038, 38145, 38148, 38160, 38208, 38296, 38305, 38400, 38470, 38500, 38913, 38916, 38928, 38950, 38976,+ 39081, 39168, 39241, 39250, 39568, 40960, 40965, 40970, 40980, 40994, 41002, 41025, 41028, 41040, 41122, 41130,+ 41280, 41317, 41474, 41482, 41506, 41512, 41514, 41602, 41608, 41610, 41640, 41985, 41988, 42000, 42048, 42121,+ 42148, 42240, 42265, 42577, 43018, 43048, 43170, 43348, 43398, 43528, 43530, 43552, 43554, 43560, 43656, 43690,+ };++ const int kmap_size = 43692;+ //const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2;+ const int nwant = type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2;+ const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 :+ type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 :+ type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? kgrid_1bit_2048 : kgrid_2bit_1024;+ uint64_t * kgrid_q2xs;+ int * kmap_q2xs;+ uint16_t * kneighbors_q2xs;++ //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);+ uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t));+ for (int k = 0; k < grid_size; ++k) {+ int8_t * pos = (int8_t *)(the_grid + k);+ for (int i = 0; i < 8; ++i) {+ int l = (kgrid[k] >> 2*i) & 0x3;+ pos[i] = 2*l + 1;+ }+ }+ kgrid_q2xs = the_grid;+ iq2_data[gindex].grid = the_grid;+ kmap_q2xs = (int *)malloc(kmap_size*sizeof(int));+ iq2_data[gindex].map = kmap_q2xs;+ for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1;+ uint64_t aux64;+ uint8_t * aux8 = (uint8_t *)&aux64;+ for (int i = 0; i < grid_size; ++i) {+ aux64 = kgrid_q2xs[i];+ uint16_t index = 0;+ for (int k=0; k<8; ++k) {+ uint16_t q = (aux8[k] - 1)/2;+ index |= (q << 2*k);+ }+ kmap_q2xs[index] = i;+ }+ // The neighbour search runs in three passes:+ // 1. Parallel: for each i, qsort and count its neighbours into n_per_i,+ // and reduce the totals (num_neighbors, num_not_in_map).+ // 2. Serial: prefix-sum n_per_i into offsets[], so each i has a+ // pre-assigned slice of kneighbors_q2xs to write into.+ // 3. Parallel: redo the qsort and write each i's neighbour list at+ // offsets[i].+ int * n_per_i = (int *)malloc(kmap_size*sizeof(int));+ GGML_ASSERT(n_per_i);+ int num_neighbors = 0, num_not_in_map = 0;+#ifdef GGML_USE_OPENMP+ #pragma omp parallel reduction(+:num_neighbors,num_not_in_map)+#endif+ {+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));+ GGML_ASSERT(dist2);+ int8_t pos[8];+ int i;+#ifdef GGML_USE_OPENMP+ #pragma omp for schedule(dynamic, 64)+#endif+ for (i = 0; i < kmap_size; ++i) {+ if (kmap_q2xs[i] >= 0) {+ n_per_i[i] = 0;+ continue;+ }+ ++num_not_in_map;+ for (int k = 0; k < 8; ++k) {+ int l = (i >> 2*k) & 0x3;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);+ int d2 = 0;+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);+ int n = 0; int d2 = dist2[0];+ int nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ ++n;+ }+ n_per_i[i] = n;+ num_neighbors += n;+ }+ free(dist2);+ }+ //printf("%s: %d neighbours in total\n", __func__, num_neighbors);+ kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));+ iq2_data[gindex].neighbours = kneighbors_q2xs;++ int * offsets = (int *)malloc(kmap_size*sizeof(int));+ GGML_ASSERT(offsets);+ int counter = 0;+ for (int i = 0; i < kmap_size; ++i) {+ if (kmap_q2xs[i] >= 0) {+ offsets[i] = -1;+ continue;+ }+ offsets[i] = counter;+ counter += 1 + n_per_i[i];+ }++#ifdef GGML_USE_OPENMP+ #pragma omp parallel+#endif+ {+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));+ GGML_ASSERT(dist2);+ int8_t pos[8];+ int i;+#ifdef GGML_USE_OPENMP+ #pragma omp for schedule(dynamic, 64)+#endif+ for (i = 0; i < kmap_size; ++i) {+ if (kmap_q2xs[i] >= 0) continue;+ for (int k = 0; k < 8; ++k) {+ int l = (i >> 2*k) & 0x3;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);+ int d2 = 0;+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);+ int local_counter = offsets[i];+ kmap_q2xs[i] = -(local_counter + 1);+ int d2 = dist2[0];+ uint16_t * start = &kneighbors_q2xs[local_counter++];+ int n = 0, nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ kneighbors_q2xs[local_counter++] = dist2[2*j+1];+ ++n;+ }+ *start = n;+ }+ free(dist2);+ }+ free(offsets);+ free(n_per_i);+}++void iq2xs_free_impl(enum ggml_type type) {+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);+ const int gindex = iq2_data_index(type);+ if (iq2_data[gindex].grid) {+ free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL;+ free(iq2_data[gindex].map); iq2_data[gindex].map = NULL;+ free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL;+ }+}++static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,+ const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_d2 = FLT_MAX;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float d2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = pg[i];+ float diff = scale*q - xval[i];+ d2 += weight[i]*diff*diff;+ }+ if (d2 < best_d2) {+ best_d2 = d2; grid_index = neighbours[j];+ }+ }+ GGML_ASSERT(grid_index >= 0);+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}++static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 3;++ const int64_t nbl = n/QK_K;++ block_iq2_xxs * y = vy;++ float scales[QK_K/32];+ float weight[32];+ float xval[32];+ int8_t L[32];+ int8_t Laux[32];+ float waux[32];+ uint8_t block_signs[4];+ uint32_t q2[2*(QK_K/32)];++ for (int ibl = 0; ibl < nbl; ++ibl) {++ y[ibl].d = GGML_FP32_TO_FP16(0.f);+ memset(q2, 0, QK_K/4);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = sumx2/QK_K;++ for (int ib = 0; ib < QK_K/32; ++ib) {+ const float * xb = xbl + 32*ib;+ const float * qw = quant_weights + QK_K*ibl + 32*ib;+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 4; ++k) {+ int nflip = 0;+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);+ }+ }+ if (nflip%2) {+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];+ for (int i = 1; i < 8; ++i) {+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];+ if (ax < min) {+ min = ax; imin = i;+ }+ }+ xval[8*k+imin] = -xval[8*k+imin];+ s ^= (1 << imin);+ }+ block_signs[k] = s & 127;+ }+ float max = xval[0];+ for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);+ if (max < GROUP_MAX_EPS) {+ scales[ib] = 0;+ memset(L, 0, 32);+ continue;+ }+ float scale = make_qp_quants(32, kMaxQ+1, xval, (uint8_t*)L, weight);+ float eff_max = scale*kMaxQ;+ if (eff_max <= 0) {+ scales[ib] = 0;+ memset(L, 0, 32);+ continue;+ }+ float best = 0;+ for (int is = -6; is <= 6; ++is) {+ float id = (2*kMaxQ-1+is*0.1f)/eff_max;+ float this_scale = 1/id;+ for (int k = 0; k < 4; ++k) {+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ memcpy(L, Laux, 32);+ }+ }+ if (scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 4; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 2*i);+ }+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ }+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);+ for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)+ // and correspondingly flip quant signs.+ scale = -scale;+ for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;+ }+ for (int k = 0; k < 4; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ printf("\n");+ GGML_ABORT("fatal error");+ }+ q2[2*ib+0] |= ((uint32_t) grid_index << 8*k);+ q2[2*ib+1] |= (block_signs[k] << 7*k);+ }+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ memset(y[ibl].qs, 0, QK_K/4);+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d);+ float id = 1/d;+ for (int ib = 0; ib < QK_K/32; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ q2[2*ib+1] |= ((uint32_t)l << 28);+ }+ memcpy(y[ibl].qs, q2, QK_K/4);+ }+}++static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ2_XS);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 3;++ const int64_t nbl = n/QK_K;++ block_iq2_xs * y = vy;++ float scales[QK_K/16];+ float weight[16];+ float xval[16];+ int8_t L[16];+ int8_t Laux[16];+ float waux[16];+ bool is_on_grid[2];+ bool is_on_grid_aux[2];+ uint8_t block_signs[2];+ uint16_t q2[2*(QK_K/16)];++ for (int ibl = 0; ibl < nbl; ++ibl) {++ y[ibl].d = GGML_FP32_TO_FP16(0.f);+ memset(q2, 0, QK_K/4);+ memset(y[ibl].scales, 0, QK_K/32);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = sumx2/QK_K;++ for (int ib = 0; ib < QK_K/16; ++ib) {+ const float * xb = xbl + 16*ib;+ const float * qw = quant_weights + QK_K*ibl + 16*ib;+ for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 2; ++k) {+ int nflip = 0;+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);+ }+ }+ if (nflip%2) {+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];+ for (int i = 1; i < 8; ++i) {+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];+ if (ax < min) {+ min = ax; imin = i;+ }+ }+ xval[8*k+imin] = -xval[8*k+imin];+ s ^= (1 << imin);+ }+ block_signs[k] = s & 127;+ }+ float max = xval[0];+ for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);+ memset(L, 0, 16);+ if (max < GROUP_MAX_EPS) {+ scales[ib] = 0;+ continue;+ }+ float best = 0;+ float scale = max/(2*kMaxQ-1);+ is_on_grid[0] = is_on_grid[1] = true;+ for (int is = -9; is <= 9; ++is) {+ float id = (2*kMaxQ-1+is*0.1f)/max;+ float this_scale = 1/id;+ for (int k = 0; k < 2; ++k) {+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ is_on_grid_aux[k] = true;+ if (grid_index < 0) {+ is_on_grid_aux[k] = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ for (int i = 0; i < 16; ++i) L[i] = Laux[i];+ for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];+ }+ }+ int n_not_ongrid = 0;+ for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ if (n_not_ongrid > 0 && scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 2; ++k) {+ if (is_on_grid[k]) continue;+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 2*i);+ L[8*k + i] = l;+ }+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ scale = -scale;+ for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127;+ }+ for (int k = 0; k < 2; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ printf("\n");+ GGML_ABORT("fatal error");+ }+ q2[2*ib+k] = grid_index | (block_signs[k] << 9);+ }+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ memset(y[ibl].qs, 0, QK_K/4);+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d);+ float id = 1/d;+ for (int ib = 0; ib < QK_K/16; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ if (ib%2 == 0) y[ibl].scales[ib/2] = l;+ else y[ibl].scales[ib/2] |= (l << 4);+ }+ memcpy(y[ibl].qs, q2, QK_K/4);++ }+}++size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ int64_t nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq2_xxs);+ }+ return nrow * nblock * sizeof(block_iq2_xxs);+}++size_t quantize_iq2_xs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ int64_t nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq2_xs);+ }+ return nrow * nblock * sizeof(block_iq2_xs);+}++//+// ============================================= 3-bit using D4 lattice+//++typedef struct {+ uint32_t * grid;+ int * map;+ uint16_t * neighbours;+} iq3_entry_t;++static iq3_entry_t iq3_data[2] = {+ {NULL, NULL, NULL},+ {NULL, NULL, NULL},+};++static inline int iq3_data_index(int grid_size) {+ (void)grid_size;+ GGML_ASSERT(grid_size == 256 || grid_size == 512);+ return grid_size == 256 ? 0 : 1;+}++static int iq3_compare_func(const void * left, const void * right) {+ const int * l = (const int *)left;+ const int * r = (const int *)right;+ return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;+}++void iq3xs_init_impl(int grid_size) {+ const int gindex = iq3_data_index(grid_size);+ if (iq3_data[gindex].grid) {+ return;+ }+ static const uint16_t kgrid_256[256] = {+ 0, 2, 4, 9, 11, 15, 16, 18, 25, 34, 59, 61, 65, 67, 72, 74,+ 81, 85, 88, 90, 97, 108, 120, 128, 130, 132, 137, 144, 146, 153, 155, 159,+ 169, 175, 189, 193, 199, 200, 202, 213, 248, 267, 287, 292, 303, 315, 317, 321,+ 327, 346, 362, 413, 436, 456, 460, 462, 483, 497, 513, 515, 520, 522, 529, 531,+ 536, 538, 540, 551, 552, 576, 578, 585, 592, 594, 641, 643, 648, 650, 657, 664,+ 698, 704, 706, 720, 729, 742, 758, 769, 773, 808, 848, 852, 870, 889, 901, 978,+ 992, 1024, 1026, 1033, 1035, 1040, 1042, 1046, 1049, 1058, 1089, 1091, 1093, 1096, 1098, 1105,+ 1112, 1139, 1143, 1144, 1152, 1154, 1161, 1167, 1168, 1170, 1183, 1184, 1197, 1217, 1224, 1228,+ 1272, 1276, 1309, 1323, 1347, 1367, 1377, 1404, 1473, 1475, 1486, 1509, 1537, 1544, 1546, 1553,+ 1555, 1576, 1589, 1594, 1600, 1602, 1616, 1625, 1636, 1638, 1665, 1667, 1672, 1685, 1706, 1722,+ 1737, 1755, 1816, 1831, 1850, 1856, 1862, 1874, 1901, 1932, 1950, 1971, 2011, 2032, 2052, 2063,+ 2077, 2079, 2091, 2095, 2172, 2192, 2207, 2208, 2224, 2230, 2247, 2277, 2308, 2345, 2356, 2389,+ 2403, 2424, 2501, 2504, 2506, 2520, 2570, 2593, 2616, 2624, 2630, 2646, 2669, 2700, 2714, 2746,+ 2754, 2795, 2824, 2835, 2839, 2874, 2882, 2905, 2984, 3028, 3042, 3092, 3108, 3110, 3124, 3153,+ 3185, 3215, 3252, 3288, 3294, 3364, 3397, 3434, 3483, 3523, 3537, 3587, 3589, 3591, 3592, 3610,+ 3626, 3670, 3680, 3722, 3749, 3754, 3776, 3789, 3803, 3824, 3857, 3873, 3904, 3906, 3924, 3992,+ };+ static const uint16_t kgrid_512[512] = {+ 0, 1, 2, 5, 7, 8, 9, 10, 12, 14, 16, 17, 21, 27, 32, 34,+ 37, 39, 41, 43, 48, 50, 57, 60, 63, 64, 65, 66, 68, 72, 73, 77,+ 80, 83, 87, 89, 93, 100, 113, 117, 122, 128, 129, 133, 135, 136, 139, 142,+ 145, 149, 152, 156, 162, 165, 167, 169, 171, 184, 187, 195, 201, 205, 208, 210,+ 217, 219, 222, 228, 232, 234, 247, 249, 253, 256, 267, 271, 273, 276, 282, 288,+ 291, 297, 312, 322, 324, 336, 338, 342, 347, 353, 357, 359, 374, 379, 390, 393,+ 395, 409, 426, 441, 448, 450, 452, 464, 466, 470, 475, 488, 492, 512, 513, 514,+ 516, 520, 521, 523, 525, 527, 528, 530, 537, 540, 542, 556, 558, 561, 570, 576,+ 577, 579, 582, 584, 588, 593, 600, 603, 609, 616, 618, 632, 638, 640, 650, 653,+ 655, 656, 660, 666, 672, 675, 685, 688, 698, 705, 708, 711, 712, 715, 721, 727,+ 728, 732, 737, 754, 760, 771, 773, 778, 780, 793, 795, 802, 806, 808, 812, 833,+ 840, 843, 849, 856, 858, 873, 912, 916, 919, 932, 934, 961, 963, 968, 970, 977,+ 989, 993, 1010, 1016, 1024, 1025, 1027, 1029, 1031, 1032, 1034, 1036, 1038, 1041, 1043, 1047,+ 1048, 1050, 1057, 1059, 1061, 1064, 1066, 1079, 1080, 1083, 1085, 1088, 1090, 1096, 1099, 1103,+ 1106, 1109, 1113, 1116, 1122, 1129, 1153, 1156, 1159, 1169, 1171, 1176, 1183, 1185, 1195, 1199,+ 1209, 1212, 1216, 1218, 1221, 1225, 1234, 1236, 1241, 1243, 1250, 1256, 1270, 1281, 1287, 1296,+ 1299, 1306, 1309, 1313, 1338, 1341, 1348, 1353, 1362, 1375, 1376, 1387, 1400, 1408, 1410, 1415,+ 1425, 1453, 1457, 1477, 1481, 1494, 1496, 1507, 1512, 1538, 1545, 1547, 1549, 1551, 1554, 1561,+ 1563, 1565, 1570, 1572, 1575, 1577, 1587, 1593, 1601, 1603, 1605, 1612, 1617, 1619, 1632, 1648,+ 1658, 1662, 1664, 1674, 1680, 1690, 1692, 1704, 1729, 1736, 1740, 1745, 1747, 1751, 1752, 1761,+ 1763, 1767, 1773, 1787, 1795, 1801, 1806, 1810, 1817, 1834, 1840, 1844, 1857, 1864, 1866, 1877,+ 1882, 1892, 1902, 1915, 1934, 1953, 1985, 1987, 2000, 2002, 2013, 2048, 2052, 2058, 2064, 2068,+ 2071, 2074, 2081, 2088, 2104, 2114, 2119, 2121, 2123, 2130, 2136, 2141, 2147, 2153, 2157, 2177,+ 2179, 2184, 2189, 2193, 2203, 2208, 2223, 2226, 2232, 2244, 2249, 2251, 2256, 2258, 2265, 2269,+ 2304, 2306, 2324, 2335, 2336, 2361, 2373, 2375, 2385, 2418, 2443, 2460, 2480, 2504, 2509, 2520,+ 2531, 2537, 2562, 2568, 2572, 2578, 2592, 2596, 2599, 2602, 2614, 2620, 2625, 2627, 2629, 2634,+ 2641, 2650, 2682, 2688, 2697, 2707, 2712, 2718, 2731, 2754, 2759, 2760, 2775, 2788, 2793, 2805,+ 2811, 2817, 2820, 2832, 2842, 2854, 2890, 2902, 2921, 2923, 2978, 3010, 3012, 3026, 3081, 3083,+ 3085, 3097, 3099, 3120, 3136, 3152, 3159, 3188, 3210, 3228, 3234, 3245, 3250, 3256, 3264, 3276,+ 3281, 3296, 3349, 3363, 3378, 3392, 3395, 3420, 3440, 3461, 3488, 3529, 3531, 3584, 3588, 3591,+ 3600, 3602, 3614, 3616, 3628, 3634, 3650, 3657, 3668, 3683, 3685, 3713, 3716, 3720, 3726, 3729,+ 3736, 3753, 3778, 3802, 3805, 3819, 3841, 3845, 3851, 3856, 3880, 3922, 3938, 3970, 3993, 4032,+ };++ const int kmap_size = 4096;+ const int nwant = grid_size == 256 ? 2 : 3;+ const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512;+ uint32_t * kgrid_q3xs;+ int * kmap_q3xs;+ uint16_t * kneighbors_q3xs;++ //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);+ uint32_t * the_grid = (uint32_t *)malloc(grid_size*sizeof(uint32_t));+ for (int k = 0; k < grid_size; ++k) {+ int8_t * pos = (int8_t *)(the_grid + k);+ for (int i = 0; i < 4; ++i) {+ int l = (kgrid[k] >> 3*i) & 0x7;+ pos[i] = 2*l + 1;+ }+ }+ kgrid_q3xs = the_grid;+ iq3_data[gindex].grid = the_grid;+ kmap_q3xs = (int *)malloc(kmap_size*sizeof(int));+ iq3_data[gindex].map = kmap_q3xs;+ for (int i = 0; i < kmap_size; ++i) kmap_q3xs[i] = -1;+ uint32_t aux32;+ uint8_t * aux8 = (uint8_t *)&aux32;+ for (int i = 0; i < grid_size; ++i) {+ aux32 = kgrid_q3xs[i];+ uint16_t index = 0;+ for (int k=0; k<4; ++k) {+ uint16_t q = (aux8[k] - 1)/2;+ index |= (q << 3*k);+ }+ kmap_q3xs[index] = i;+ }+ // See explanation of parallelism in iq2xs_init_impl+ int * n_per_i = (int *)malloc(kmap_size*sizeof(int));+ GGML_ASSERT(n_per_i);+ int num_neighbors = 0, num_not_in_map = 0;+#ifdef GGML_USE_OPENMP+ #pragma omp parallel reduction(+:num_neighbors,num_not_in_map)+#endif+ {+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));+ GGML_ASSERT(dist2);+ int8_t pos[4];+ int i;+#ifdef GGML_USE_OPENMP+ #pragma omp for schedule(dynamic, 64)+#endif+ for (i = 0; i < kmap_size; ++i) {+ if (kmap_q3xs[i] >= 0) {+ n_per_i[i] = 0;+ continue;+ }+ ++num_not_in_map;+ for (int k = 0; k < 4; ++k) {+ int l = (i >> 3*k) & 0x7;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);+ int d2 = 0;+ for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);+ int n = 0; int d2 = dist2[0];+ int nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ ++n;+ }+ n_per_i[i] = n;+ num_neighbors += n;+ }+ free(dist2);+ }+ //printf("%s: %d neighbours in total\n", __func__, num_neighbors);+ kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));+ iq3_data[gindex].neighbours = kneighbors_q3xs;++ int * offsets = (int *)malloc(kmap_size*sizeof(int));+ GGML_ASSERT(offsets);+ int counter = 0;+ for (int i = 0; i < kmap_size; ++i) {+ if (kmap_q3xs[i] >= 0) {+ offsets[i] = -1;+ continue;+ }+ offsets[i] = counter;+ counter += 1 + n_per_i[i];+ }++#ifdef GGML_USE_OPENMP+ #pragma omp parallel+#endif+ {+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));+ GGML_ASSERT(dist2);+ int8_t pos[4];+ int i;+#ifdef GGML_USE_OPENMP+ #pragma omp for schedule(dynamic, 64)+#endif+ for (i = 0; i < kmap_size; ++i) {+ if (kmap_q3xs[i] >= 0) continue;+ for (int k = 0; k < 4; ++k) {+ int l = (i >> 3*k) & 0x7;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);+ int d2 = 0;+ for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);+ int local_counter = offsets[i];+ kmap_q3xs[i] = -(local_counter + 1);+ int d2 = dist2[0];+ uint16_t * start = &kneighbors_q3xs[local_counter++];+ int n = 0, nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ kneighbors_q3xs[local_counter++] = dist2[2*j+1];+ ++n;+ }+ *start = n;+ }+ free(dist2);+ }+ free(offsets);+ free(n_per_i);+}++void iq3xs_free_impl(int grid_size) {+ GGML_ASSERT(grid_size == 256 || grid_size == 512);+ const int gindex = iq3_data_index(grid_size);+ if (iq3_data[gindex].grid) {+ free(iq3_data[gindex].grid); iq3_data[gindex].grid = NULL;+ free(iq3_data[gindex].map); iq3_data[gindex].map = NULL;+ free(iq3_data[gindex].neighbours); iq3_data[gindex].neighbours = NULL;+ }+}++static int iq3_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint32_t * GGML_RESTRICT grid,+ const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_d2 = FLT_MAX;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float d2 = 0;+ for (int i = 0; i < 4; ++i) {+ float q = pg[i];+ float diff = scale*q - xval[i];+ d2 += weight[i]*diff*diff;+ }+ if (d2 < best_d2) {+ best_d2 = d2; grid_index = neighbours[j];+ }+ }+ GGML_ASSERT(grid_index >= 0);+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 4; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}++static void quantize_row_iq3_xxs_impl(int grid_size, const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n,+ const float * GGML_RESTRICT quant_weights) {++ const int gindex = iq3_data_index(grid_size);++ const uint32_t * kgrid_q3xs = iq3_data[gindex].grid;+ const int * kmap_q3xs = iq3_data[gindex].map;+ const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours;++ //GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 8;++ const int64_t nbl = n/QK_K;++ ggml_fp16_t * dh;+ uint8_t * qs;+ int block_size;+ if (grid_size == 256) {+ block_iq3_xxs * y = vy;+ dh = &y->d;+ qs = y->qs;+ block_size = sizeof(block_iq3_xxs);+ } else {+ block_iq3_s * y = vy;+ dh = &y->d;+ qs = y->qs;+ block_size = sizeof(block_iq3_s);+ }+ int quant_size = block_size - sizeof(ggml_fp16_t);++ float scales[QK_K/32];+ float weight[32];+ float xval[32];+ int8_t L[32];+ int8_t Laux[32];+ float waux[32];+ bool is_on_grid[8];+ bool is_on_grid_aux[8];+ uint8_t block_signs[8];+ uint8_t q3[3*(QK_K/8)+QK_K/32];+ uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K/4);+ uint8_t * qh = q3 + 3*(QK_K/8);++ for (int ibl = 0; ibl < nbl; ++ibl) {++ dh[0] = GGML_FP32_TO_FP16(0.f);+ memset(q3, 0, 3*QK_K/8+QK_K/32);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = 2*sumx2/QK_K;++ for (int ib = 0; ib < QK_K/32; ++ib) {+ const float * xb = xbl + 32*ib;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*ibl + 32*ib;+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ } else {+ for (int i = 0; i < 32; ++i) weight[i] = xb[i]*xb[i];+ }+ for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 4; ++k) {+ int nflip = 0;+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);+ }+ }+ if (nflip%2) {+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];+ for (int i = 1; i < 8; ++i) {+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];+ if (ax < min) {+ min = ax; imin = i;+ }+ }+ xval[8*k+imin] = -xval[8*k+imin];+ s ^= (1 << imin);+ }+ block_signs[k] = s & 127;+ }+ float max = xval[0];+ for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);+ memset(L, 0, 32);+ if (max < GROUP_MAX_EPS_IQ3_XXS) {+ scales[ib] = 0;+ continue;+ }+ float best = 0;+ float scale = max/(2*kMaxQ-1);+ for (int k = 0; k < 8; ++k) is_on_grid[k] = true;+ for (int is = -15; is <= 15; ++is) {+ float id = (2*kMaxQ-1+is*0.2f)/max;+ float this_scale = 1/id;+ for (int k = 0; k < 8; ++k) {+ for (int i = 0; i < 4; ++i) {+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));+ Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i);+ int grid_index = kmap_q3xs[u];+ is_on_grid_aux[k] = true;+ if (grid_index < 0) {+ is_on_grid_aux[k] = false;+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ for (int i = 0; i < 32; ++i) L[i] = Laux[i];+ for (int k = 0; k < 8; ++k) is_on_grid[k] = is_on_grid_aux[k];+ }+ }+ int n_not_ongrid = 0;+ for (int k = 0; k < 8; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ if (n_not_ongrid > 0 && scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 8; ++k) {+ if (is_on_grid[k]) continue;+ uint16_t u = 0;+ for (int i = 0; i < 4; ++i) {+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 3*i);+ }+ int grid_index = kmap_q3xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k);+ }+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index);+ for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2;+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)+ // and correspondingly flip quant signs.+ scale = -scale;+ for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;+ }+ for (int k = 0; k < 8; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i);+ int grid_index = kmap_q3xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]);+ printf("\n");+ GGML_ABORT("fatal error");+ }+ if (grid_size == 256) {+ q3[8*ib+k] = grid_index;+ } else {+ q3[8*ib+k] = grid_index & 255;+ qh[ib] |= ((grid_index >> 8) << k);+ }++ }+ scales_and_signs[ib] = block_signs[0] | (block_signs[1] << 7) | (block_signs[2] << 14) | (block_signs[3] << 21);+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ memset(qs, 0, quant_size);+ dh += block_size/sizeof(ggml_fp16_t);+ qs += block_size;+ continue;+ }++ float d = max_scale/31;+ dh[0] = GGML_FP32_TO_FP16(d * 1.0125f); // small improvement via this fudge factor+ float id = 1/d;+ for (int ib = 0; ib < QK_K/32; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ scales_and_signs[ib] |= ((uint32_t)l << 28);+ }+ memcpy(qs, q3, quant_size);++ dh += block_size/sizeof(ggml_fp16_t);+ qs += block_size;++ }+}++size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ int64_t nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_iq3_xxs_impl(256, src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq3_xxs);+ }+ return nrow * nblock * sizeof(block_iq3_xxs);+}++void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ quantize_row_iq3_xxs_impl(256, x, y, k, NULL);+}++static void quantize_row_iq3_s_impl(int block_size, const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int n,+ const float * GGML_RESTRICT quant_weights,+ float * scales,+ float * weight,+ float * xval,+ int8_t * L,+ int8_t * Laux,+ float * waux,+ bool * is_on_grid,+ bool * is_on_grid_aux,+ uint8_t * block_signs) {++ const int gindex = iq3_data_index(512);++ const uint32_t * kgrid_q3xs = iq3_data[gindex].grid;+ const int * kmap_q3xs = iq3_data[gindex].map;+ const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours;++ //GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 8;++ const int64_t nbl = n/QK_K;++ block_iq3_s * y = vy;++ const int bs4 = block_size/4;+ const int bs8 = block_size/8;++ for (int ibl = 0; ibl < nbl; ++ibl) {++ memset(&y[ibl], 0, sizeof(block_iq3_s));+ y[ibl].d = GGML_FP32_TO_FP16(0.f);++ uint8_t * qs = y[ibl].qs;+ uint8_t * qh = y[ibl].qh;+ uint8_t * signs = y[ibl].signs;++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = 2*sumx2/QK_K;++ for (int ib = 0; ib < QK_K/block_size; ++ib) {+ const float * xb = xbl + block_size*ib;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ } else {+ for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];+ }+ for (int i = 0; i < block_size; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < bs8; ++k) {+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; s |= (1 << i);+ }+ }+ block_signs[k] = s;+ }+ float max = xval[0];+ for (int i = 1; i < block_size; ++i) max = MAX(max, xval[i]);+ memset(L, 0, block_size);+ if (!max) {+ scales[ib] = 0;+ continue;+ }+ float best = 0;+ float scale = max/(2*kMaxQ-1);+ for (int k = 0; k < bs4; ++k) is_on_grid[k] = false;+ for (int is = -9; is <= 9; ++is) {+ float id = (2*kMaxQ-1+is*0.2f)/max;+ float this_scale = 1/id;+ for (int k = 0; k < bs4; ++k) {+ for (int i = 0; i < 4; ++i) {+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));+ Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i);+ int grid_index = kmap_q3xs[u];+ is_on_grid_aux[k] = true;+ if (grid_index < 0) {+ is_on_grid_aux[k] = false;+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < block_size; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ for (int i = 0; i < block_size; ++i) L[i] = Laux[i];+ for (int k = 0; k < bs4; ++k) is_on_grid[k] = is_on_grid_aux[k];+ }+ }+ int n_not_ongrid = 0;+ for (int k = 0; k < bs4; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ if (n_not_ongrid > 0 && scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < bs4; ++k) {+ //if (is_on_grid[k]) continue;+ uint16_t u = 0;+ for (int i = 0; i < 4; ++i) {+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 3*i);+ }+ int grid_index = kmap_q3xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k);+ }+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index);+ for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2;+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < block_size; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)+ // and correspondingly flip quant signs.+ scale = -scale;+ for (int k = 0; k < bs8; ++k) block_signs[k] = ~block_signs[k];+ }+ for (int k = 0; k < bs4; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i);+ int grid_index = kmap_q3xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]);+ printf("\n");+ GGML_ABORT("fatal error");+ }+ qs[k] = grid_index & 255;+ qh[(ib*bs4+k)/8] |= ((grid_index >> 8) << ((ib*bs4+k)%8));+ }+ qs += bs4;+ for (int k = 0; k < bs8; ++k) signs[k] = block_signs[k];+ signs += bs8;+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f);+ float id = 1/d;+ for (int ib = 0; ib < QK_K/block_size; ib += 2) {+ int l1 = nearest_int(0.5f*(id*scales[ib+0]-1));+ l1 = MAX(0, MIN(15, l1));+ int l2 = nearest_int(0.5f*(id*scales[ib+1]-1));+ l2 = MAX(0, MIN(15, l2));+ y[ibl].scales[ib/2] = l1 | (l2 << 4);+ }++ }+}++#define IQ3S_BLOCK_SIZE 32+size_t quantize_iq3_s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ int64_t nblock = n_per_row/QK_K;+ float scales[QK_K/IQ3S_BLOCK_SIZE];+ float weight[IQ3S_BLOCK_SIZE];+ float xval[IQ3S_BLOCK_SIZE];+ int8_t L[IQ3S_BLOCK_SIZE];+ int8_t Laux[IQ3S_BLOCK_SIZE];+ float waux[IQ3S_BLOCK_SIZE];+ bool is_on_grid[IQ3S_BLOCK_SIZE/4];+ bool is_on_grid_aux[IQ3S_BLOCK_SIZE/4];+ uint8_t block_signs[IQ3S_BLOCK_SIZE/8];+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_iq3_s_impl(IQ3S_BLOCK_SIZE, src, qrow, n_per_row, quant_weights,+ scales, weight, xval, L, Laux, waux, is_on_grid, is_on_grid_aux, block_signs);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq3_s);+ }+ return nrow * nblock * sizeof(block_iq3_s);+}++void quantize_row_iq3_s_ref(const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ quantize_iq3_s(x, y, 1, k, NULL);+}+++// =================================== 1.5 bpw ===================================================++static int iq1_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,+ const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float * scale, int8_t * GGML_RESTRICT L, int ngrid) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_score = -FLT_MAX;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = (pg[i] - 3)/2;+ float w = weight[i];+ sumqx += w*q*xval[i];+ sumq2 += w*q*q;+ }+ if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ *scale = sumqx/sumq2; best_score = *scale * sumqx;+ grid_index = neighbours[j];+ }+ }+ if (grid_index < 0) {+ for (int i = 0; i < ngrid; ++i) {+ const int8_t * grid_i = (const int8_t *)(grid + i);+ float sumqx = 0, sumq2 = 0;+ for (int j = 0; j < 8; ++j) {+ float w = weight[j];+ float q = (grid_i[j] - 3)/2;+ sumqx += w*q*xval[j];+ sumq2 += w*q*q;+ }+ if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ *scale = sumqx/sumq2; best_score = *scale*sumqx;+ grid_index = i;+ }+ }+ }+ if (grid_index < 0) {+ printf("Oops, did not find grid point\n");+ printf("Have %d neighbours\n", num_neighbors);+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = (pg[i] - 3)/2;+ float w = weight[i];+ sumqx += w*q*xval[i];+ sumq2 += w*q*q;+ }+ printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2);+ }+ }+ GGML_ASSERT(grid_index >= 0);+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+ *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result.+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}++static int iq1_find_best_neighbour2(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,+ const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, const float * GGML_RESTRICT xg, int8_t * GGML_RESTRICT L, int ngrid) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_score = FLT_MAX;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float d2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = xg[(pg[i] - 1)/2];+ float w = weight[i];+ float diff = scale*q - xval[i];+ d2 += w*diff*diff;+ }+ if (d2 < best_score) {+ best_score = d2;+ grid_index = neighbours[j];+ }+ }+ if (grid_index < 0) {+ for (int i = 0; i < ngrid; ++i) {+ const int8_t * grid_i = (const int8_t *)(grid + i);+ float d2 = 0;+ for (int j = 0; j < 8; ++j) {+ float w = weight[j];+ float q = xg[(grid_i[j] - 1)/2];+ float diff = scale*q - xval[i];+ d2 += w*diff*diff;+ }+ if (d2 < best_score) {+ best_score = d2;+ grid_index = i;+ }+ }+ }+ if (grid_index < 0) {+ printf("Oops, did not find grid point\n");+ printf("Have %d neighbours\n", num_neighbors);+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = xg[(pg[i] - 1)/2];+ float w = weight[i];+ sumqx += w*q*xval[i];+ sumq2 += w*q*q;+ }+ printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2);+ }+ }+ GGML_ASSERT(grid_index >= 0);+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}++static int iq1_sort_helper(const void * left, const void * right) {+ const float * l = left;+ const float * r = right;+ return *l < *r ? -1 : *l > *r ? 1 : 0;+}++#define IQ1S_BLOCK_SIZE 32+#define IQ1M_BLOCK_SIZE 16+static void quantize_row_iq1_s_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights,+ float * scales,+ float * weight,+ float * sumx,+ float * sumw,+ float * pairs,+ int8_t * L,+ uint16_t * index,+ int8_t * shifts) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ1_S);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ block_iq1_s * y = vy;++ const int64_t nbl = n/QK_K;++ const int block_size = IQ1S_BLOCK_SIZE;++ const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA};+ const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA};+++ int * idx = (int *)(pairs + 1);++ for (int ibl = 0; ibl < nbl; ++ibl) {++ y[ibl].d = GGML_FP32_TO_FP16(0.f);+ memset(y[ibl].qs, 0, QK_K/8);+ memset(y[ibl].qh, 0, QK_K/16);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = 2*sumx2/QK_K;++ for (int ib = 0; ib < QK_K/block_size; ++ib) {+ const float * xb = xbl + block_size*ib;+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ float max = fabsf(xb[0]);+ for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));+ if (max < GROUP_MAX_EPS_IQ1_S) {+ scales[ib] = 0;+ shifts[ib] = 1;+ memset(L, 1, block_size);+ continue;+ }+ // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.+ // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two+ // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights+ // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and+ // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale+ // for each possible and score for each split.+ for (int j = 0; j < block_size; ++j) {+ pairs[2*j] = xb[j];+ idx[2*j] = j;+ }+ qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper);+ {+ sumx[0] = sumw[0] = 0;+ for (int j = 0; j < block_size; ++j) {+ int i = idx[2*j];+ sumx[j+1] = sumx[j] + weight[i]*xb[i];+ sumw[j+1] = sumw[j] + weight[i];+ }+ }+ float best_score = -FLT_MAX, scale = max;+ int besti1 = -1, besti2 = -1, best_shift = 0;+ for (int i1 = 0; i1 <= block_size; ++i1) {+ for (int i2 = i1; i2 <= block_size; ++i2) {+ float sumqx = (sumx[i1] - sumx[0])*x_p[0] + (sumx[i2] - sumx[i1])*x_p[1] + (sumx[block_size] - sumx[i2])*x_p[2];+ float sumq2 = (sumw[i1] - sumw[0])*x_p[0]*x_p[0] + (sumw[i2] - sumw[i1])*x_p[1]*x_p[1] + (sumw[block_size] - sumw[i2])*x_p[2]*x_p[2];+ if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ scale = sumqx/sumq2; best_score = scale*sumqx;+ besti1 = i1; besti2 = i2; best_shift = 1;+ }+ sumqx = (sumx[i1] - sumx[0])*x_m[0] + (sumx[i2] - sumx[i1])*x_m[1] + (sumx[block_size] - sumx[i2])*x_m[2];+ sumq2 = (sumw[i1] - sumw[0])*x_m[0]*x_m[0] + (sumw[i2] - sumw[i1])*x_m[1]*x_m[1] + (sumw[block_size] - sumw[i2])*x_m[2]*x_m[2];+ if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ scale = sumqx/sumq2; best_score = scale*sumqx;+ besti1 = i1; besti2 = i2; best_shift = -1;+ }+ }+ }+ if (besti1 < 0 || besti2 < 0 || best_shift == 0) {+ scales[ib] = 0;+ shifts[ib] = 1;+ memset(L, 1, block_size);+ continue;+ }+ for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;+ for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;+ for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;+ if (scale < 0) {+ for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j];+ scale = -scale; best_shift = -best_shift;+ }+ bool all_on_grid = true;+ const float * xx = best_shift == 1 ? x_p : x_m;+ for (int k = 0; k < block_size/8; ++k) {+ uint16_t u = 0;+ for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ all_on_grid = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S);+ GGML_ASSERT(grid_index >= 0);+ }+ index[k] = grid_index;+ }+ if (!all_on_grid) {+ float sumqx = 0, sumq2 = 0;+ for (int k = 0; k < block_size/8; ++k) {+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]);+ for (int j = 0; j < 8; ++j) {+ float w = weight[8*k + j];+ float q = xx[(pg[j] - 1)/2];+ sumqx += w*q*xb[8*k+j];+ sumq2 += w*q*q;+ }+ }+ if (sumqx > 0 && sumq2 > 0) scale = sumqx/sumq2;+ }+ uint16_t h = 0;+ for (int k = 0; k < block_size/8; ++k) {+ y[ibl].qs[(block_size/8)*ib + k] = index[k] & 255;+ h |= (index[k] >> 8) << 3*k;+ }+ y[ibl].qh[ib] = h;+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ shifts[ib] = best_shift;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ continue;+ }++ float d = max_scale/15;+ y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.125f is another fudge factor. Don't ask me why it is needed.+ float id = 1/d;+ for (int ib = 0; ib < QK_K/block_size; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(7, l));+ if (shifts[ib] == -1) l |= 8;+ y[ibl].qh[ib] |= (l << 12);+ }+ }+}++size_t quantize_iq1_s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ float scales[QK_K/IQ1S_BLOCK_SIZE];+ float weight[IQ1S_BLOCK_SIZE];+ int8_t L[IQ1S_BLOCK_SIZE];+ float sumx[IQ1S_BLOCK_SIZE+1];+ float sumw[IQ1S_BLOCK_SIZE+1];+ float pairs[2*IQ1S_BLOCK_SIZE];+ uint16_t index[IQ1S_BLOCK_SIZE/8];+ int8_t shifts[QK_K/IQ1S_BLOCK_SIZE];+ int64_t nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_iq1_s_impl(src, qrow, n_per_row, quant_weights, scales, weight, sumx, sumw, pairs, L, index, shifts);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq1_s);+ }+ return nrow * nblock * sizeof(block_iq1_s);+}++static void quantize_row_iq1_m_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights,+ float * scales,+ float * weight,+ float * pairs,+ int8_t * L,+ uint16_t * index,+ int8_t * shifts) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ1_M);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ //GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ block_iq1_m * y = vy;++ const int64_t nbl = n/QK_K;++ const int block_size = IQ1M_BLOCK_SIZE;++ const float x_p[3] = {-1 + IQ1M_DELTA, IQ1M_DELTA, 1 + IQ1M_DELTA};+ const float x_m[3] = {-1 - IQ1M_DELTA, -IQ1M_DELTA, 1 - IQ1M_DELTA};+ const uint8_t masks[4] = {0x00, 0x80, 0x08, 0x88};++ int * idx = (int *)(pairs + 1);++ float sumqx[4], sumq2[4];++ iq1m_scale_t s;+ const float * xx;++ for (int ibl = 0; ibl < nbl; ++ibl) {+ memset(y[ibl].qs, 0, QK_K/8);+ memset(y[ibl].qh, 0, QK_K/16);+ memset(y[ibl].scales, 0, QK_K/32);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = 2*sumx2/QK_K;++ for (int ib = 0; ib < QK_K/block_size; ++ib) {+ const float * xb = xbl + block_size*ib;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ } else {+ for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];+ }+ float max = fabsf(xb[0]);+ for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));+ if (max < GROUP_MAX_EPS_IQ1_M) {+ scales[ib] = 0;+ shifts[ib] = 0;+ memset(L, 1, block_size);+ continue;+ }+ // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.+ // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two+ // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights+ // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and+ // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale+ // for each possible and score for each split.+ for (int j = 0; j < block_size; ++j) {+ pairs[2*j] = xb[j];+ idx[2*j] = j;+ }+ qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper);+ float best_score = -FLT_MAX, scale = max;+ int besti1 = -1, besti2 = -1, best_k = -1;+ // 0: +, ++ // 1: +, -+ // 2: -, ++ // 3: -, -+ // prefix sums are kept per half of the block because each half can use a different sign (x_p or x_m)+ // since v[0]-v[1] = v[1]-v[2] = -1 for both x_p and x_m, the 3-group sum for a split collapses to T*v[2] - px[i1] - px[i2]+ {+ float px[2][IQ1M_BLOCK_SIZE+1];+ float pw[2][IQ1M_BLOCK_SIZE+1];+ px[0][0] = px[1][0] = 0;+ pw[0][0] = pw[1][0] = 0;+ for (int j = 0; j < block_size; ++j) {+ const int i = idx[2*j];+ const int h = i < block_size/2 ? 0 : 1;+ px[h][j+1] = px[h][j] + weight[i]*xb[i];+ px[1-h][j+1] = px[1-h][j];+ pw[h][j+1] = pw[h][j] + weight[i];+ pw[1-h][j+1] = pw[1-h][j];+ }+ const float txs[2] = {px[0][block_size], px[1][block_size]}; // total weight*x per half+ const float tws[2] = {pw[0][block_size], pw[1][block_size]}; // total weight per half+ const float p2 = x_p[2], m2 = x_m[2];+ const float cp1 = x_p[0]*x_p[0] - x_p[1]*x_p[1];+ const float cp2 = x_p[1]*x_p[1] - x_p[2]*x_p[2];+ const float cm1 = x_m[0]*x_m[0] - x_m[1]*x_m[1];+ const float cm2 = x_m[1]*x_m[1] - x_m[2]*x_m[2];+ for (int i1 = 0; i1 <= block_size; ++i1) {+ for (int i2 = i1; i2 <= block_size; ++i2) {+ float qx_p[2], qx_m[2], q2_p[2], q2_m[2];+ for (int h = 0; h < 2; ++h) {+ const float sx = px[h][i1] + px[h][i2];+ qx_p[h] = txs[h]*p2 - sx;+ qx_m[h] = txs[h]*m2 - sx;+ q2_p[h] = tws[h]*p2*p2 + pw[h][i1]*cp1 + pw[h][i2]*cp2;+ q2_m[h] = tws[h]*m2*m2 + pw[h][i1]*cm1 + pw[h][i2]*cm2;+ }+ sumqx[0] = qx_p[0] + qx_p[1];+ sumqx[1] = qx_p[0] + qx_m[1];+ sumqx[2] = qx_m[0] + qx_p[1];+ sumqx[3] = qx_m[0] + qx_m[1];+ sumq2[0] = q2_p[0] + q2_p[1];+ sumq2[1] = q2_p[0] + q2_m[1];+ sumq2[2] = q2_m[0] + q2_p[1];+ sumq2[3] = q2_m[0] + q2_m[1];+ for (int k = 0; k < 4; ++k) {+ if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) {+ scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k];+ besti1 = i1; besti2 = i2; best_k = k;+ }+ }+ }+ }+ }+ if (besti1 < 0 || besti2 < 0 || best_k < 0) {+ scales[ib] = 0;+ shifts[ib] = 0;+ memset(L, 1, block_size);+ continue;+ }+ for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;+ for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;+ for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;+ if (scale < 0) {+ for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j];+ scale = -scale;+ best_k = best_k == 0 ? 3 : best_k == 1 ? 2 : best_k == 2 ? 1 : 0;+ }+ bool all_on_grid = true;+ for (int k = 0; k < block_size/8; ++k) {+ if (k == 0) xx = best_k < 2 ? x_p : x_m;+ else xx = best_k%2 == 0 ? x_p : x_m;+ uint16_t u = 0;+ for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ all_on_grid = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S);+ GGML_ASSERT(grid_index >= 0);+ }+ index[k] = grid_index;+ }+ if (!all_on_grid) {+ float sumqx_f = 0, sumq2_f = 0;+ for (int k = 0; k < block_size/8; ++k) {+ if (k == 0) xx = best_k < 2 ? x_p : x_m;+ else xx = best_k%2 == 0 ? x_p : x_m;+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]);+ for (int j = 0; j < 8; ++j) {+ float w = weight[8*k + j];+ float q = xx[(pg[j] - 1)/2];+ sumqx_f += w*q*xb[8*k+j];+ sumq2_f += w*q*q;+ }+ }+ if (sumqx_f > 0 && sumq2_f > 0) scale = sumqx_f/sumq2_f;+ }+ y[ibl].qs[2*ib + 0] = index[0] & 255;+ y[ibl].qs[2*ib + 1] = index[1] & 255;+ y[ibl].qh[ib] = (index[0] >> 8) | ((index[1] >> 8) << 4);+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ shifts[ib] = best_k;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ continue;+ }++ uint16_t * sc = (uint16_t *)y[ibl].scales;+ float d = max_scale/15;+ float id = 1/d;+ float sumqx_f = 0, sumq2_f = 0;+ for (int ib = 0; ib < QK_K/block_size; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib+0]-1));+ l = MAX(0, MIN(7, l));+ sc[ib/4] |= (l << 3*(ib%4));+ y[ibl].qh[ib] |= masks[shifts[ib]];+ const float * xb = xbl + block_size*ib;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*ibl + block_size*ib;+ for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ } else {+ for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];+ }+ for (int k = 0; k < block_size/8; ++k) {+ if (k == 0) xx = shifts[ib] < 2 ? x_p : x_m;+ else xx = shifts[ib]%2 == 0 ? x_p : x_m;+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + y[ibl].qs[2*ib+k] + ((y[ibl].qh[ib] << (8 - 4*k)) & 0x700));+ for (int j = 0; j < 8; ++j) {+ float w = weight[8*k + j];+ float q = xx[(pg[j] - 1)/2]*(2*l+1);+ sumqx_f += w*q*xb[8*k+j];+ sumq2_f += w*q*q;+ }+ }+ }+ if (sumq2_f > 0) d = sumqx_f/sumq2_f;+ s.f16 = GGML_FP32_TO_FP16(d*1.1125f); // 1.1125f is another fudge factor. Don't ask me why it is needed.+ sc[0] |= ((s.u16 & 0x000f) << 12);+ sc[1] |= ((s.u16 & 0x00f0) << 8);+ sc[2] |= ((s.u16 & 0x0f00) << 4);+ sc[3] |= ((s.u16 & 0xf000) << 0);+ }+}++size_t quantize_iq1_m(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ float scales[QK_K/IQ1M_BLOCK_SIZE];+ float weight[IQ1M_BLOCK_SIZE];+ int8_t L[IQ1M_BLOCK_SIZE];+ float pairs[2*IQ1M_BLOCK_SIZE];+ uint16_t index[IQ1M_BLOCK_SIZE/8];+ int8_t shifts[QK_K/IQ1M_BLOCK_SIZE];+ int64_t nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_iq1_m_impl(src, qrow, n_per_row, quant_weights, scales, weight, pairs, L, index, shifts);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq1_m);+ }+ return nrow * nblock * sizeof(block_iq1_m);+}++// ============================ 4-bit non-linear quants++static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * GGML_RESTRICT x,+ ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l,+ float * scales, float * weight, uint8_t * L,+ const int8_t * values,+ const float * quant_weights,+ const int ntry) {++ float sigma2 = 0;+ for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j];+ sigma2 *= 2.f/super_block_size;++ memset(q4, 0, super_block_size/2);+ dh[0] = GGML_FP32_TO_FP16(0.f);++ float max_scale = 0, amax_scale = 0;+ for (int ib = 0; ib < super_block_size/block_size; ++ib) {+ const float * xb = x + ib*block_size;+ uint8_t * Lb = L + ib*block_size;+ if (quant_weights) {+ const float * qw = quant_weights + ib*block_size;+ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ } else {+ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j];+ }+ float amax = 0, max = 0;+ for (int j = 0; j < block_size; ++j) {+ float ax = fabsf(xb[j]);+ if (ax > amax) {+ amax = ax; max = xb[j];+ }+ }+ if (amax < GROUP_MAX_EPS) {+ scales[ib] = 0;+ continue;+ }+ float d = ntry > 0 ? -max/values[0] : max/values[0];+ float id = 1/d;+ float sumqx = 0, sumq2 = 0;+ for (int j = 0; j < block_size; ++j) {+ float al = id*xb[j];+ int l = best_index_int8(16, values, al);+ Lb[j] = l;+ float q = values[l];+ float w = weight[j];+ sumqx += w*q*xb[j];+ sumq2 += w*q*q;+ }+ d = sumq2 > 0 ? sumqx/sumq2 : 0.f;+ float best = d*sumqx;+ for (int itry = -ntry; itry <= ntry; ++itry) {+ id = (itry + values[0])/max;+ sumqx = sumq2 = 0;+ for (int j = 0; j < block_size; ++j) {+ float al = id*xb[j];+ int l = best_index_int8(16, values, al);+ float q = values[l];+ float w = weight[j];+ sumqx += w*q*xb[j];+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ d = sumqx/sumq2; best = d * sumqx;+ }+ }+ scales[ib] = d;+ float abs_d = fabsf(d);+ if (abs_d > amax_scale) {+ amax_scale = abs_d; max_scale = d;+ }+ }++ if (super_block_size/block_size > 1) {+ int nb = super_block_size/block_size;+ memset(scales_h, 0, ((nb+7)/8)*sizeof(uint16_t));+ float d = -max_scale/32;+ dh[0] = GGML_FP32_TO_FP16(d);+ float id = d ? 1/d : 0.f;+ for (int ib = 0; ib < super_block_size/block_size; ++ib) {+ int l = nearest_int(id*scales[ib]);+ l = MAX(-32, MIN(31, l));+ float dl = d * l;+ float idl = dl ? 1/dl : 0.f;+ uint8_t * Lb = L + ib*block_size;+ const float * xb = x + ib*block_size;+ for (int j = 0; j < block_size; ++j) {+ Lb[j] = best_index_int8(16, values, idl*xb[j]);+ }+ l += 32;+ uint8_t l_l = l & 0xf;+ uint8_t l_h = l >> 4;+ if (ib%2 == 0) scales_l[ib/2] = l_l;+ else scales_l[ib/2] |= (l_l << 4);+ scales_h[ib/8] |= (l_h << 2*(ib%8));+ }+ } else {+ dh[0] = GGML_FP32_TO_FP16(scales[0]);+ if (ntry > 0) {+ float id = scales[0] ? 1/scales[0] : 0;+ for (int j = 0; j < super_block_size; ++j) {+ L[j] = best_index_int8(16, values, id*x[j]);+ }+ }+ }++ for (int i = 0; i < super_block_size/32; ++i) {+ for (int j = 0; j < 16; ++j) {+ q4[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4);+ }+ }+}++size_t quantize_iq4_nl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK4_NL == 0);+ int64_t nblock = n_per_row/QK4_NL;+ char * qrow = (char *)dst;+ uint8_t L[QK4_NL];+ float weight[QK4_NL];+ uint16_t unused_h;+ uint8_t * unused_l = NULL;+ float scale;+ for (int64_t row = 0; row < nrow; ++row) {+ block_iq4_nl * iq4 = (block_iq4_nl *)qrow;+ for (int ibl = 0; ibl < nblock; ++ibl) {+ const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL;+ quantize_row_iq4_nl_impl(QK4_NL, 32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,+ &scale, weight, L, kvalues_iq4nl, qw, 7);+ }+ src += n_per_row;+ qrow += nblock*sizeof(block_iq4_nl);+ }+ return nrow * nblock * sizeof(block_iq4_nl);+}++//void quantize_row_iq4_nl_ref(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {+void quantize_row_iq4_nl_ref(const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k) {+ GGML_ASSERT(k%QK4_NL == 0);+ int64_t nblock = k/QK4_NL;+ uint8_t L[QK4_NL];+ float weight[QK4_NL];+ uint16_t unused_h;+ uint8_t * unused_l = NULL;+ float scale;+ block_iq4_nl * iq4 = y;+ for (int ibl = 0; ibl < nblock; ++ibl) {+ quantize_row_iq4_nl_impl(QK4_NL, 32, x + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,+ &scale, weight, L, kvalues_iq4nl, NULL, -1);+ }+}++size_t quantize_iq4_xs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ int64_t nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ uint8_t L[QK_K];+ float weight[32];+ float scales[QK_K/32];+ for (int64_t row = 0; row < nrow; ++row) {+ block_iq4_xs * iq4 = (block_iq4_xs *)qrow;+ for (int ibl = 0; ibl < nblock; ++ibl) {+ const float * qw = quant_weights ? quant_weights + QK_K*ibl : NULL;+ quantize_row_iq4_nl_impl(QK_K, 32, src + QK_K*ibl, &iq4[ibl].d, iq4[ibl].qs, &iq4[ibl].scales_h, iq4[ibl].scales_l,+ scales, weight, L, kvalues_iq4nl, qw, 7);+ }+ src += n_per_row;+ qrow += nblock*sizeof(block_iq4_xs);+ }+ return nrow * nblock * sizeof(block_iq4_xs);+}++void quantize_row_iq4_xs_ref(const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ quantize_iq4_xs(x, y, 1, k, NULL);+}++// =============================== 2.5625 bpw++static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ2_S);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 3;++ const int64_t nbl = n/QK_K;++ block_iq2_s * y = vy;++ float scales[QK_K/16];+ float weight[16];+ float xval[16];+ int8_t L[16];+ int8_t Laux[16];+ float waux[16];+ bool is_on_grid[2];+ bool is_on_grid_aux[2];+ uint8_t block_signs[2];++ for (int ibl = 0; ibl < nbl; ++ibl) {++ memset(&y[ibl], 0, sizeof(block_iq2_s));+ y[ibl].d = GGML_FP32_TO_FP16(0.f);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = 2*sumx2/QK_K;++ for (int ib = 0; ib < QK_K/16; ++ib) {+ const float * xb = xbl + 16*ib;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*ibl + 16*ib;+ for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ } else {+ for (int i = 0; i < 16; ++i) weight[i] = 0.25f*sigma2 + xb[i]*xb[i];+ }+ for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 2; ++k) {+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; s |= (1 << i);+ }+ }+ block_signs[k] = s;+ }+ float max = xval[0];+ for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);+ memset(L, 0, 16);+ if (max < GROUP_MAX_EPS_IQ2_S) {+ scales[ib] = 0;+ continue;+ }+ float best = 0;+ float scale = max/(2*kMaxQ-1);+ is_on_grid[0] = is_on_grid[1] = true;+ for (int is = -9; is <= 9; ++is) {+ float id = (2*kMaxQ-1+is*0.1f)/max;+ float this_scale = 1/id;+ for (int k = 0; k < 2; ++k) {+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ is_on_grid_aux[k] = true;+ if (grid_index < 0) {+ is_on_grid_aux[k] = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ for (int i = 0; i < 16; ++i) L[i] = Laux[i];+ for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];+ }+ }+ int n_not_ongrid = 0;+ for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ if (n_not_ongrid > 0 && scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 2; ++k) {+ if (is_on_grid[k]) continue;+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 2*i);+ L[8*k + i] = l;+ }+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ scale = -scale;+ for (int k = 0; k < 2; ++k) block_signs[k] = ~block_signs[k];+ }+ for (int k = 0; k < 2; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ printf("\n");+ GGML_ABORT("fatal error");+ }+ const int i8 = 2*ib + k;+ y[ibl].qs[i8] = grid_index & 255;+ y[ibl].qh[i8/4] |= ((grid_index >> 8) << 2*(i8%4));+ y[ibl].qs[QK_K/8 + i8] = block_signs[k];+ }+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d * 0.9875f);+ float id = 1/d;+ for (int ib = 0; ib < QK_K/16; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ if (ib%2 == 0) y[ibl].scales[ib/2] = l;+ else y[ibl].scales[ib/2] |= (l << 4);+ }+ }+}++size_t quantize_iq2_s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ GGML_ASSERT(n_per_row%QK_K == 0);+ int64_t nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_iq2_s_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq2_s);+ }+ return nrow * nblock * sizeof(block_iq2_s);+}++void quantize_row_iq2_s_ref(const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k) {+ assert(k % QK_K == 0);+ quantize_iq2_s(x, y, 1, k, NULL);+}++// =============================== data validation++static bool validate_float(float f, size_t i) {+ if (isinf(f)) {+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);+ return false;+ }++ if (isnan(f)) {+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);+ return false;+ }++ return true;+}++static bool isinf_fp16(ggml_fp16_t f) {+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) == 0;+}++static bool isnan_fp16(ggml_fp16_t f) {+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) != 0;+}++static bool validate_fp16(ggml_fp16_t f, size_t i) {+ if (isinf_fp16(f)) {+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);+ return false;+ }++ if (isnan_fp16(f)) {+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);+ return false;+ }++ return true;+}++static bool validate_e_e8m0(uint8_t e, size_t i) {+ if (e == 0xff) {+ fprintf(stderr, "ggml_validate_row_data: found invalid e value %d at block %zu\n", e, i);+ return false;+ }++ return true;+}++#define VALIDATE_ROW_DATA_D_F16_IMPL(type, data, nb) \+ const type * q = (const type *) (data); \+ for (size_t i = 0; i < (nb); ++i) { \+ if (!validate_fp16(q[i].d, i)) { \+ return false; \+ } \+ }++#define VALIDATE_ROW_DATA_DM_F16_IMPL(type, data, nb, d, m) \+ const type * q = (const type *) (data); \+ for (size_t i = 0; i < (nb); ++i) { \+ if (!validate_fp16(q[i].d, i) || !validate_fp16(q[i].m, i)) { \+ return false; \+ } \+ }++#define VALIDATE_ROW_DATA_E_E8M0_IMPL(type, data, nb) \+ const type * q = (const type *) (data); \+ for (size_t i = 0; i < (nb); ++i) { \+ if (!validate_e_e8m0(q[i].e, i)) { \+ return false; \+ } \+ }++#define VALIDATE_ROW_DATA_DVEC_F16_IMPL(type, data, nb, nr) \+ const type * q = (const type *) (data); \+ for (size_t i = 0; i < (nb); ++i) { \+ for (size_t j = 0; j < (nr); ++j) { \+ if (!validate_fp16(q[i].d[j], i)) { \+ return false; \+ } \+ } \+ }++bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) {+ if (type < 0 || type >= GGML_TYPE_COUNT) {+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);+ return false;+ }++ if (nbytes % ggml_type_size(type) != 0) {+ fprintf(stderr, "%s: invalid size %zu for type %s (type size = %zu)\n", __func__, nbytes, ggml_type_name(type), ggml_type_size(type));+ return false;+ }++ const size_t nb = nbytes/ggml_type_size(type);++ switch (type) {+ case GGML_TYPE_BF16:+ {+ int nans = 0;+ int infs = 0;+ const unsigned short * f = (const unsigned short *) data;+ for (size_t i = 0; i < nb; ++i) {+ nans += (f[i] & 0x7fff) > 0x7f80;+ infs += (f[i] & 0x7fff) == 0x7f80;+ }+ if (nans) {+ fprintf(stderr, "%s: found %d NaNs in row of %zu BF16 values\n", __func__, nans, nb);+ return false;+ }+ if (infs) {+ fprintf(stderr, "%s: found %d infinities in row of %zu BF16 values\n", __func__, infs, nb);+ return false;+ }+ } break;+ case GGML_TYPE_F16:+ {+ const ggml_fp16_t * f = (const ggml_fp16_t *) data;+ size_t i = 0;+#if defined(__AVX2__)+ for (; i + 15 < nb; i += 16) {+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi16(0x7c00));+ __m256i cmp = _mm256_cmpeq_epi16(vexp, _mm256_set1_epi16(0x7c00));+ int mask = _mm256_movemask_epi8(cmp);+ if (mask) {+ for (size_t j = 0; j < 16; ++j) {+ if (!validate_fp16(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#elif defined(__ARM_NEON)+ for (; i + 7 < nb; i += 8) {+ uint16x8_t v = vld1q_u16(f + i);+ uint16x8_t vexp = vandq_u16(v, vdupq_n_u16(0x7c00));+ uint16x8_t cmp = vceqq_u16(vexp, vdupq_n_u16(0x7c00));+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(cmp, 4)), 0);+ if (mask) {+ for (size_t j = 0; j < 8; ++j) {+ if (!validate_fp16(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#endif+ for (; i < nb; ++i) {+ if (!validate_fp16(f[i], i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_F32:+ {+ const float * f = (const float *) data;+ size_t i = 0;+#if defined(__AVX2__)+ for (; i + 7 < nb; i += 8) {+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi32(0x7f800000));+ __m256i cmp = _mm256_cmpeq_epi32(vexp, _mm256_set1_epi32(0x7f800000));+ int mask = _mm256_movemask_epi8(cmp);+ if (mask) {+ for (size_t j = 0; j < 8; ++j) {+ if (!validate_float(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#elif defined(__ARM_NEON)+ for (; i + 3 < nb; i += 4) {+ uint32x4_t v = vld1q_u32((const uint32_t *)f + i);+ uint32x4_t vexp = vandq_u32(v, vdupq_n_u32(0x7f800000));+ uint32x4_t cmp = vceqq_u32(vexp, vdupq_n_u32(0x7f800000));+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u16(vshrn_n_u32(cmp, 8)), 0);+ if (mask) {+ for (size_t j = 0; j < 4; ++j) {+ if (!validate_float(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#endif+ for (; i < nb; ++i) {+ if (!validate_float(f[i], i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_F64:+ {+ const double * f = (const double *) data;+ for (size_t i = 0; i < nb; ++i) {+ if (!validate_float(f[i], i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_Q1_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q1_0, data, nb);+ } break;+ case GGML_TYPE_Q2_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q2_0, data, nb);+ } break;+ case GGML_TYPE_Q4_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb);+ } break;+ case GGML_TYPE_Q4_1:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_1, data, nb, d, m);+ } break;+ case GGML_TYPE_Q5_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_0, data, nb);+ } break;+ case GGML_TYPE_Q5_1:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_1, data, nb, d, m);+ } break;+ case GGML_TYPE_Q8_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q8_0, data, nb);+ } break;+ case GGML_TYPE_MXFP4:+ {+ VALIDATE_ROW_DATA_E_E8M0_IMPL(block_mxfp4, data, nb);+ } break;+ case GGML_TYPE_NVFP4:+ {+ // UE4M3 scales are uint8_t — all byte values are valid+ GGML_UNUSED(data);+ GGML_UNUSED(nb);+ } break;+ case GGML_TYPE_Q2_K:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin);+ } break;+ case GGML_TYPE_Q3_K:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_K, data, nb);+ } break;+ case GGML_TYPE_Q4_K:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d, dmin);+ } break;+ case GGML_TYPE_Q5_K:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_K, data, nb, d, dmin);+ } break;+ case GGML_TYPE_Q6_K:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q6_K, data, nb);+ } break;+ case GGML_TYPE_Q8_K:+ {+ const block_q8_K * q = (const block_q8_K *) data;+ for (size_t i = 0; i < nb; ++i) {+ if (!validate_float(q[i].d, i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_TQ1_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_tq1_0, data, nb);+ } break;+ case GGML_TYPE_TQ2_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_tq2_0, data, nb);+ } break;+ case GGML_TYPE_IQ1_S:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb);+ } break;+ case GGML_TYPE_IQ1_M:+ {+ const block_iq1_m * q = (const block_iq1_m *) data;+ for (size_t i = 0; i < nb; ++i) {+ iq1m_scale_t scale;+ const uint16_t * sc = (const uint16_t *)q[i].scales;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);+ if (!validate_fp16(scale.f16, i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_IQ2_XXS:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xxs, data, nb);+ } break;+ case GGML_TYPE_IQ2_XS:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xs, data, nb);+ } break;+ case GGML_TYPE_IQ2_S:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_s, data, nb);+ } break;+ case GGML_TYPE_IQ3_XXS:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_xxs, data, nb);+ } break;++ case GGML_TYPE_IQ3_S:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_s, data, nb);+ } break;+ case GGML_TYPE_IQ4_XS:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_xs, data, nb);+ } break;+ case GGML_TYPE_IQ4_NL:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);+ } break;++ case GGML_TYPE_I8:+ case GGML_TYPE_I16:+ case GGML_TYPE_I32:+ case GGML_TYPE_I64:+ // nothing to validate+ break;+ default:+ {+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);+ return false;+ }+ }++ return true;+}diff --git a/python/freetoken/kernel/csrc/ggml/src/ggml-quants.h b/python/freetoken/kernel/csrc/ggml/src/ggml-quants.hnew file mode 100644index 00000000..75188f1a--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/src/ggml-quants.h@@ -0,0 +1,115 @@+#pragma once++#define GGML_COMMON_DECL_C+#include "ggml-common.h"++#include "ggml.h"++// GGML internal header++#ifdef __cplusplus+extern "C" {+#endif++// NOTE: these functions are defined as GGML_API because they used by the CPU backend++// Quantization+GGML_API void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q5_1_ref(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);++GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k);++GGML_API void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);++GGML_API void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k);++GGML_API void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_iq4_nl_ref (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_iq4_xs_ref (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_iq3_s_ref (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);++// Dequantization+GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+//GGML_API void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++GGML_API void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++GGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++GGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")+GGML_API size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++GGML_API size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++GGML_API size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++GGML_API void iq2xs_init_impl(enum ggml_type type);+GGML_API void iq2xs_free_impl(enum ggml_type type);+GGML_API void iq3xs_init_impl(int grid_size);+GGML_API void iq3xs_free_impl(int grid_size);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/ggml_cpu/__init__.py b/python/freetoken/kernel/ggml_cpu/__init__.pynew file mode 100644index 00000000..909c973e--- /dev/null+++ b/python/freetoken/kernel/ggml_cpu/__init__.py@@ -0,0 +1,37 @@+"""ggml-cpu row kernels for GGUF experts on the CPU MoE executor.++setup.py builds one module per x86 variant of llama.cpp's GGML_CPU_ALL_VARIANTS; select() picks the one this CPU runs best.+"""++from __future__ import annotations++import functools+import importlib+import os+import pkgutil+from typing import NamedTuple+++class Variant(NamedTuple):+ name: str+ api: int # address of the variant's kernel table, for _cpu_moe.set_ggml_cpu+++def select() -> Variant | None:+ """The highest-scoring variant this CPU runs (upstream's rule), or the one FREETOKEN_GGML_CPU_VARIANT names; None when the build has none."""+ names = sorted(m.name[1:] for m in pkgutil.iter_modules(__path__) if m.name.startswith("_"))+ forced = os.environ.get("FREETOKEN_GGML_CPU_VARIANT")+ if forced:+ if forced not in names:+ raise RuntimeError(f"FREETOKEN_GGML_CPU_VARIANT={forced}: this build has the ggml-cpu variants {names}")+ names = [forced]+ best, best_score = None, 0+ for name in names:+ # importing a variant runs no code built for its ISA extensions+ mod = importlib.import_module(f"{__name__}._{name}")+ if (score := mod.score()) > best_score:+ best, best_score = Variant(name, mod.api()), score+ if forced and best is None:+ raise RuntimeError(f"FREETOKEN_GGML_CPU_VARIANT={forced}: this CPU lacks an ISA extension the variant is built for")+ return bestdiff --git a/python/freetoken/layers/quantization/moe/gguf.py b/python/freetoken/layers/quantization/moe/gguf.pyindex dfd545be..416c2f4b 100644--- a/python/freetoken/layers/quantization/moe/gguf.py+++ b/python/freetoken/layers/quantization/moe/gguf.py@@ -142,13 +142,9 @@ class GgmlMoEKernel(MoEKernel):class GGUFMoEMethod(MoEMethod):candidates = (GgmlMoEKernel,)- @property- def cpu_format(self) -> str | None:- # the CPU executor reads Q4_0 experts in their GGUF rows and BF16 experts as its bf16 banks; other ggml types have no CPU kernel yet+ def cpu_spec(self) -> tuple[str | None, dict[str, int]]:gate_up, down = expert_types(self.cfg)- names = {type_name(gate_up), type_name(down)}- if names == {"Q4_0"}:- return "q4_0"- if names == {"BF16"}:- return "bf16"- return self.kernel.cpu_format+ # BF16 experts keep the executor's bf16 kernels, which use AVX512-BF16 where the CPU has it; ggml-cpu is built for AVX2 only+ if float_dtype(gate_up) is torch.bfloat16 and float_dtype(down) is torch.bfloat16:+ return "bf16", {}+ return "gguf", {"gate_up_type": gate_up, "down_type": down}diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex 86a03f41..f7ea5aa3 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -63,7 +63,7 @@ from freetoken.utils import init_loggerlogger = init_logger(__name__)# Formats the CPU MoE C++ kernel can compute AND this bench can build banks for; anything-# else is offload-only here. (The kernel also does q4_0, but this bench has no q4_0 banks.)+# else is offload-only here._CPU_MOE_FORMATS = frozenset({"bf16", "nvfp4", "mxfp4_triton", "ds_fp4", "fp8_block"})# Formats this bench can build synthetic (correctly-sized) banks for._BUILDABLE_FORMATS = frozenset({"bf16", "nvfp4", "fp8_block", "mxfp4_triton", "ds_fp4"})diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex 5d9bd3c9..4c0aee7a 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -147,6 +147,16 @@ def resolve_threads_and_affinity(requested: int) -> tuple[int, list[int]]:return len(reps), list(reps)+def _format_label(name: str, opts: tuple) -> str:+ """One format instance as the startup log names it, e.g. ``gguf(gate_up=IQ2_XXS, down=IQ4_NL)``."""+ if name == "gguf":+ from freetoken.layers.quantization.ggml_types import type_name++ gate_up, down = dict(opts)["gate_up_type"], dict(opts)["down_type"]+ return f"gguf(gate_up={type_name(gate_up)}, down={type_name(down)})"+ return name + (f"({', '.join(f'{k}={v}' for k, v in opts)})" if opts else "")++class CpuMoeExecutor:"""Decode-time CPU expert compute over an ``OffloadMoeCache``'s host banks(the formats ``_cpu_moe.weight_formats()`` lists — see ``_resolve_banks``)."""@@ -182,6 +192,15 @@ class CpuMoeExecutor:f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are "f"{f!r}; use --moe-strategy offload (GPU-side dequant) instead.")+ if f == "gguf":+ from freetoken.kernel import ggml_cpu++ if (variant := ggml_cpu.select()) is None:+ raise NotImplementedError(+ "--moe-strategy cpu/hybrid cannot compute GGUF experts here: this build has no ggml-cpu kernels (x86-64 only); "+ "use --moe-strategy offload instead."+ )+ _cpu_moe.set_ggml_cpu(variant.api)if activation not in _ACT_IDS:raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")if apply_router_weight_on_input and (output_only := {"mxfp4_triton", "ds_fp4"} & set(names)):@@ -348,7 +367,7 @@ class CpuMoeExecutor:self._watchdog.start()- fmt_label = "; ".join(f"{name}{dict(opts) if opts else ''}: {n} layers" for (name, opts), n in layer_counts.items())+ fmt_label = "; ".join(f"{_format_label(name, opts)}: {n} layers" for (name, opts), n in layer_counts.items())logger.info_rank0(f"CPU MoE executor ready: threads={nthreads} (pinned to cores "f"{core_ids[0]}..{core_ids[-1]}) isa={self.isa} "@@ -404,6 +423,8 @@ class CpuMoeExecutor:(see ``_make_table``), not a single bank's -- the C++ side resolves``tbl[layer_id]`` per task."""+ if fmt == "gguf":+ return self._resolve_gguf_banks(banks, rep, int(opts["gate_up_type"]), int(opts["down_type"]))if fmt == "bf16":gate_up = banks["gate_up"]down = banks["down"]@@ -418,9 +439,6 @@ class CpuMoeExecutor:banks = self._tables(gate_up=gate_up, down=down)return banks, {}, (H, I)- if fmt == "q4_0":- return self._resolve_q4_0_banks(banks, rep)-if fmt == "mxfp4_triton":return self._resolve_mxfp4_banks(banks, rep)@@ -447,23 +465,18 @@ class CpuMoeExecutor:banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, gate_up_global=gug, down_scale=dns, down_global=dng)return banks, {}, (H, I)- def _resolve_q4_0_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:- """Native GGUF Q4_0 schema (gemma4 GGUF): per-32 blocks (fp16 scale + 16 nibble- bytes), row-major over K -- the *same* packed banks the GPU offload path streams.- gate_up is [S, 2I, H//32*18], down is [S, H, I//32*18]; the C++ W4A16 GEMV reads a- row in place (18 bytes / 32 K) and dequantizes weights inside the K-loop."""+ def _resolve_gguf_banks(self, banks: dict, rep: int, gate_up_type: int, down_type: int) -> tuple[dict, dict, tuple[int, int]]:+ """GGUF rows read in place: gate_up [E, 2I, row_bytes(H)] (gate rows, then up rows) and down [E, H, row_bytes(I)], one ggml type each; float rows keep their dtype."""+ from freetoken.layers.quantization.ggml_types import float_dtype, row_bytes, type_name+gate_up, down = banks["gate_up"], banks["down"]- assert gate_up[rep].dtype == torch.uint8 and down[rep].dtype == torch.uint8, (- gate_up[rep].dtype, down[rep].dtype,- )- I = int(gate_up[rep].shape[1] // 2)- H = int(down[rep].shape[1])- assert gate_up[rep].shape[1] == 2 * I- assert H % 32 == 0 and I % 32 == 0, (H, I)- assert int(gate_up[rep].shape[2]) == (H // 32) * 18, (gate_up[rep].shape, H)- assert int(down[rep].shape[2]) == (I // 32) * 18, (down[rep].shape, I)+ H, I = int(down[rep].shape[1]), int(gate_up[rep].shape[1] // 2)+ for role, t, ggml_type, k in (("gate_up", gate_up[rep], gate_up_type, H), ("down", down[rep], down_type, I)):+ dtype = float_dtype(ggml_type) if float_dtype(ggml_type) is not None else torch.uint8+ if t.dtype != dtype or t.shape[-1] * t.element_size() != row_bytes(ggml_type, k):+ raise ValueError(f"MoE layer {rep} {role} bank {tuple(t.shape)} {t.dtype} is not {type_name(ggml_type)} rows of {k} values")banks = self._tables(gate_up=gate_up, down=down)- return banks, {}, (H, I)+ return banks, {"gate_up_type": gate_up_type, "down_type": down_type}, (H, I)def _resolve_mxfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:"""gpt-oss mxfp4 ``mxfp4_triton`` schema: transposed split-K blocks/scalesdiff --git a/setup.py b/setup.pyindex b9683731..070b8fd5 100644--- a/setup.py+++ b/setup.py@@ -2,11 +2,13 @@ from __future__ import annotationsimport importlib.utilimport os+import platformfrom pathlib import Pathimport sys-from setuptools import setup+from setuptools import Extension, setup+from setuptools.command.build_clib import build_clibfrom torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CppExtension, ROCM_HOME@@ -96,8 +98,154 @@ else:_check_toolchain()+# ggml-cpu's row kernels for GGUF experts on the CPU MoE executor: one module per x86 variant of llama.cpp's GGML_CPU_ALL_VARIANTS (ggml/src/CMakeLists.txt), freetoken.kernel.ggml_cpu._<variant>.+# freetoken.kernel.ggml_cpu.select() imports the best one this CPU runs; each module's entry is built without the variant's ISA flags so that check is safe on any CPU.+GGML_DIR = "python/freetoken/kernel/csrc/ggml"+GGML_CPU = platform.machine().lower() in ("x86_64", "amd64")+_HASWELL = ("SSE42", "AVX", "F16C", "FMA", "AVX2", "BMI2")+GGML_CPU_VARIANTS = {+ "x64": (),+ "sse42": ("SSE42",),+ "sandybridge": ("SSE42", "AVX"),+ "ivybridge": ("SSE42", "AVX", "F16C"),+ "piledriver": ("SSE42", "AVX", "F16C", "FMA"),+ "haswell": _HASWELL,+ "skylakex": (*_HASWELL, "AVX512"),+ "cannonlake": (*_HASWELL, "AVX512", "AVX512_VBMI"),+ "cascadelake": (*_HASWELL, "AVX512", "AVX512_VNNI"),+ "icelake": (*_HASWELL, "AVX512", "AVX512_VBMI", "AVX512_VNNI"),+ "cooperlake": (*_HASWELL, "AVX512", "AVX512_VNNI", "AVX512_BF16"),+ "zen4": (*_HASWELL, "AVX512", "AVX512_VBMI", "AVX512_VNNI", "AVX512_BF16"),+ "alderlake": (*_HASWELL, "AVX_VNNI"),+ "sapphirerapids": (*_HASWELL, "AVX512", "AVX512_VBMI", "AVX512_VNNI", "AVX512_BF16", "AMX_TILE", "AMX_INT8"),+}+_GGML_GCC_FLAGS = {+ "SSE42": ["-msse4.2"], "F16C": ["-mf16c"], "FMA": ["-mfma"], "BMI2": ["-mbmi2"], "AVX": ["-mavx"], "AVX2": ["-mavx2"],+ "AVX_VNNI": ["-mavxvnni"], "AVX512": ["-mavx512f", "-mavx512cd", "-mavx512vl", "-mavx512dq", "-mavx512bw"],+ "AVX512_VBMI": ["-mavx512vbmi"], "AVX512_VNNI": ["-mavx512vnni"], "AVX512_BF16": ["-mavx512bf16"],+ "AMX_TILE": ["-mamx-tile"], "AMX_INT8": ["-mamx-int8"],+}+++def _ggml_variant_flags(feats: tuple[str, ...]) -> tuple[list[str], list[str]]:+ """(ISA flags, macros) of one variant, as upstream's ggml-cpu/CMakeLists.txt sets them; the macros also tell cpu-feats.cpp what to check."""+ if sys.platform != "win32":+ return [f for feat in feats for f in _GGML_GCC_FLAGS[feat]], [f"GGML_{feat}" for feat in feats]+ # cl takes one /arch: level and does not define the macros of the extensions above it+ if "AVX512" in feats:+ flags, macros = ["/arch:AVX512"], ["GGML_AVX512"]+ if "AVX512_VBMI" in feats:+ macros.append("__AVX512VBMI__")+ if "AVX512_VNNI" in feats:+ macros += ["__AVX512VNNI__", "GGML_AVX512_VNNI"]+ elif "AVX2" in feats:+ flags, macros = ["/arch:AVX2"], ["GGML_AVX2", "GGML_FMA", "GGML_F16C"]+ elif "AVX" in feats:+ flags, macros = ["/arch:AVX"], ["GGML_AVX"]+ elif "SSE42" in feats:+ flags, macros = ["/arch:SSE4.2"], ["GGML_SSE42"]+ else:+ flags, macros = [], []+ if "AVX_VNNI" in feats:+ macros += ["__AVXVNNI__", "GGML_AVX_VNNI"]+ if "BMI2" in feats:+ macros += ["__BMI2__", "GGML_BMI2"]+ return flags, macros+++def _ggml_cpu_targets() -> tuple[list, list, dict[str, list[str]]]:+ """(build_clib libraries, variant modules, the static libraries each module links in link order)."""+ if not GGML_CPU:+ return [], [], {}+ if sys.platform == "win32":+ cflags, c_std, cxx_std, define = ["/O2", "/DNDEBUG", "/utf-8", "/D_CRT_SECURE_NO_WARNINGS"], "/std:c11", "/std:c++20", "/D"+ # upstream builds these variants only with GCC or Clang+ variants = {k: v for k, v in GGML_CPU_VARIANTS.items() if k not in ("ivybridge", "piledriver", "cooperlake", "zen4", "sapphirerapids")}+ else:+ cflags, c_std, cxx_std, define = ["-O3", "-fPIC", "-fvisibility=hidden", "-DNDEBUG"], "-std=c11", "-std=c++20", "-D"+ variants = GGML_CPU_VARIANTS+ build = {+ "include_dirs": [f"{GGML_DIR}/include", f"{GGML_DIR}/src", f"{GGML_DIR}/ggml-cpu", GGML_DIR],+ "obj_deps": {"": sorted(str(p.relative_to(ROOT)) for d in ("include", "src", "ggml-cpu", "freetoken") for p in (ROOT / GGML_DIR / d).rglob("*.h"))},+ }+ # ggml-base, built once without ISA flags like upstream's; arch/x86/quants.c calls its quantize_row_*_ref+ libraries = [("freetoken_ggml_base", {"sources": [f"{GGML_DIR}/src/ggml-quants.c"], "cflags": [*cflags, c_std], **build})]+ modules, module_libs = [], {}+ for variant, feats in variants.items():+ flags, macros = _ggml_variant_flags(feats)+ defines = [define + m for m in macros]+ libraries += [+ (f"freetoken_ggml_cpu_{variant}", {+ "sources": [f"{GGML_DIR}/ggml-cpu/quants.c", f"{GGML_DIR}/ggml-cpu/arch/x86/quants.c", f"{GGML_DIR}/freetoken/cpu_kernels.c"],+ "cflags": [*cflags, *flags, *defines, c_std],+ **build,+ }),+ # the float row kernels; C++, so a library of their own+ (f"freetoken_ggml_cpu_vec_{variant}", {"sources": [f"{GGML_DIR}/ggml-cpu/vec.cpp"], "cflags": [*cflags, *flags, *defines, cxx_std], **build}),+ ]+ name = f"freetoken.kernel.ggml_cpu._{variant}"+ module_libs[name] = [f"freetoken_ggml_cpu_{variant}", f"freetoken_ggml_cpu_vec_{variant}", "freetoken_ggml_base"]+ modules.append(Extension(+ name=name,+ sources=[f"{GGML_DIR}/freetoken/variant_module.cpp"],+ # absolute: ninja compiles from the build directory+ include_dirs=[str(ROOT / d) for d in (*build["include_dirs"], f"{GGML_DIR}/freetoken")],+ extra_compile_args=[*cflags, *defines, f"{define}FT_GGML_VARIANT={variant}", cxx_std],+ ))+ return libraries, modules, module_libs+++ggml_cpu_libraries, ggml_cpu_modules, ggml_cpu_module_libs = _ggml_cpu_targets()+++class BuildClib(build_clib):+ def build_libraries(self, libraries):+ # the variants compile the same sources with different flags, so each library gets its own object directory+ build_temp = self.build_temp+ try:+ for lib in libraries:+ self.build_temp = os.path.join(build_temp, lib[0])+ super().build_libraries([lib])+ finally:+ self.build_temp = build_temp+++class BuildExt(BuildExtension):+ def run(self):+ # build_ext alone (`setup.py build_ext --inplace`) does not run build_clib+ if self.distribution.has_c_libraries():+ self.run_command("build_clib")+ # build_ext would link every build_clib library into every extension; build_extension adds each module's own+ libraries, self.distribution.libraries = self.distribution.libraries, None+ try:+ super().run()+ finally:+ self.distribution.libraries = libraries++ def build_extension(self, ext):+ if ext.name in ggml_cpu_module_libs:+ build_clib_dir = self.get_finalized_command("build_clib").build_clib+ if build_clib_dir not in ext.library_dirs:+ ext.library_dirs.append(build_clib_dir)+ for name in ggml_cpu_module_libs[ext.name]:+ if name not in ext.libraries:+ ext.libraries.append(name)+ lib = os.path.join(build_clib_dir, self.compiler.library_filename(name))+ # relink when only a static library changed+ if lib not in ext.depends:+ ext.depends.append(lib)+ # every variant module compiles the same source, so each gets its own object directory+ build_temp, self.build_temp = self.build_temp, os.path.join(self.build_temp, ext.name)+ try:+ super().build_extension(ext)+ finally:+ self.build_temp = build_temp+ return+ super().build_extension(ext)+setup(+ libraries=ggml_cpu_libraries,ext_modules=[CppExtension(name="freetoken.kernel._pinned_tensor",@@ -126,7 +274,7 @@ setup(for p in (ROOT / "python/freetoken/kernel/csrc/cpu_moe/kernels").glob("*.cpp")),],- include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs],+ include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs, str(ROOT / GGML_DIR / "freetoken")],library_dirs=runtime_library_dirs,libraries=[runtime_lib],extra_compile_args={"cxx": extra_compile + ["-pthread"], "nvcc": device_compile},@@ -142,6 +290,7 @@ setup(extra_compile_args=["-O3", "-std=c++20"],)] if sys.platform == "linux" else []),+ *ggml_cpu_modules,],- cmdclass={"build_ext": BuildExtension.with_options(use_ninja=True)},+ cmdclass={"build_clib": BuildClib, "build_ext": BuildExt.with_options(use_ninja=True)},)diff --git a/tests/README.md b/tests/README.mdindex adc87035..85865673 100644--- a/tests/README.md+++ b/tests/README.md@@ -14,7 +14,7 @@ in its filename (`kvcache/test_dsv4_pool.py`, `models/test_glm4_nvfp4.py`).| `tokenizer/` | `freetoken.tokenizer` — tokenize/detokenize request plumbing, thinking-mode resolution || `engine/` | `freetoken.engine` — cache budget planning, config resolution, backend gating || `kernels/` | `freetoken.kernel` / `freetoken.layers` — attention, rope, fused ops, JIT cache, pinned memory |-| `moe/` | `freetoken.moe` — offload cache, CPU/GPU expert kernels, quantized backends (fp8/nvfp4/mxfp4/q4_0) |+| `moe/` | `freetoken.moe` — offload cache, CPU/GPU expert kernels, quantized backends (fp8/nvfp4/mxfp4/gguf) || `models/` | `freetoken.models` — the registry, and the loading machinery every model shares (sharding, qkv/expert merge, streaming layers into banks) || `e2e/` | real-server gates: AIME 24/25/26 generation, runtime cache rebuild |diff --git a/tests/moe/test_cpu_moe_q4_0.py b/tests/moe/test_cpu_moe_q4_0.pydeleted file mode 100644index 9452f95c..00000000--- a/tests/moe/test_cpu_moe_q4_0.py+++ /dev/null@@ -1,213 +0,0 @@-"""CPU MoE executor -- native GGUF Q4_0 experts (``--moe-backend cpu``/``hybrid`` for-the gemma4 GGUF checkpoint).--The CPU W4A16 GEMV (``q4_0_dot`` in csrc/cpu_moe/kernels/q4_0.cpp) reads the *same*-packed Q4_0 banks the GPU offload path streams and dequantizes weights inside the-K-loop. We check it against the reference dequant (gguf-py's numpy dequantize) + the-production bf16 GPU decode kernel on byte-identical banks: both are W4A16, so the-only spread is weight bf16-rounding + reduction order -> tight relative tolerance.--Part 2 covers CUDA-graph capture/replay (the cudaLaunchHostFunc submit/sync nodes-must recompute from the freshly written pinned routing on each replay).-"""--from __future__ import annotations--from types import SimpleNamespace--import pytest-import torch--pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")---def _pack_q4_0(nibbles: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:- """Pack [S, OUT, K] uint8 nibble codes (0..15) + [S, OUT, K//32] fp16 scales into- the native Q4_0 row layout [S, OUT, K//32*18]: each 32-elem block is a 2-byte fp16- scale followed by 16 bytes whose byte j holds elem j (low nibble) and elem j+16- (high nibble) -- the exact layout ``dequant_q4_0`` reads back."""- S, OUT, K = nibbles.shape- nb = K // 32- blk = nibbles.reshape(S, OUT, nb, 32)- lo = blk[..., :16] # elems 0..15 -> low nibbles of bytes 0..15- hi = blk[..., 16:] # elems 16..31 -> high nibbles of bytes 0..15- packed = (lo | (hi << 4)).to(torch.uint8) # [S, OUT, nb, 16]- d_bytes = scale.to(torch.float16).view(torch.uint8).reshape(S, OUT, nb, 2)- row = torch.cat([d_bytes, packed], dim=-1) # [S, OUT, nb, 18]- return row.reshape(S, OUT, nb * 18).contiguous()---def _dequant_bank(packed: torch.Tensor, K: int, dev) -> torch.Tensor:- """[S, OUT, K//32*18] packed Q4_0 -> [S, OUT, K] bf16 (storage order == elem order)."""- from freetoken.third_party.gguf import GGMLQuantizationType, quants-- S, OUT, _ = packed.shape- values = quants.dequantize(packed.numpy(), GGMLQuantizationType.Q4_0)- return torch.from_numpy(values).reshape(S, OUT, K).to(dev, torch.bfloat16)---def _make_q4_0_cache(L, E, H, I, seed=0):- """Random but valid native Q4_0 banks for the cpu backend (pinned host tensors)."""- from freetoken.kernel.pinned import alloc_pinned_tensor-- torch.manual_seed(seed)- S = L * E-- def rows(OUT, K):- nib = torch.randint(0, 16, (S, OUT, K), dtype=torch.uint8)- scale = 0.02 + 0.03 * torch.rand(S, OUT, K // 32)- packed = _pack_q4_0(nib, scale) # [S, OUT, K//32*18]- pinned = alloc_pinned_tensor(*packed.shape, dtype=torch.uint8)- pinned.copy_(packed)- return pinned-- return SimpleNamespace(- quant_format="q4_0",- bank_sources={"gate_up": list(rows(2 * I, H).split(E)), "down": list(rows(H, I).split(E))},- num_layers=L,- num_experts=E,- decode_target="cpu",- cpu_executor=None,- )--[email protected]("bs", [1, 3, 8])-def test_cpu_decode_q4_0_matches_dequant_then_gpu(bs):- """CPU inline-dequant Q4_0 GEMV vs. canonical dequant_q4_0 + bf16 GPU decode."""- from freetoken.moe.cpu_executor import CpuMoeExecutor- from freetoken.moe.fused import fused_experts_decode_impl-- torch.manual_seed(400 + bs)- L, E, H, I, top_k = 3, 16, 2816, 704, 8 # gemma-4-26B-A4B geometry (H,I % 32 == 0)- layer = 1- dev = torch.device("cuda")- cache = _make_q4_0_cache(L, E, H, I)-- ex = CpuMoeExecutor(- cache,- top_k=top_k,- activation="gelu_tanh",- apply_router_weight_on_input=False,- num_threads=0,- max_tokens=bs,- device=dev,- )-- hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16) * 0.5- ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)- w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)-- cpu_out = ex.decode(layer, hidden, w, ids).float()- torch.cuda.synchronize()-- b = cache.bank_sources- gate_up_layer = _dequant_bank(b["gate_up"][layer], H, dev) # [E, 2I, H]- down_layer = _dequant_bank(b["down"][layer], I, dev) # [E, H, I]- gpu_out = fused_experts_decode_impl(- hidden, gate_up_layer, down_layer, w, ids.clone(), "gelu_tanh", False- ).float()-- rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)- assert rel < 2e-2, f"q4_0 bs={bs} rel err {rel.item()}"---def test_cpu_decode_q4_0_matches_ggml_mmvq():- """Sanity: the CPU W4A16 GEMV lands close to the GPU ggml MMVQ (W4A8) kernel the- offload path uses -- a looser tol since MMVQ quantizes activations to int8."""- from freetoken.kernel import ggml- from freetoken.layers.activation import gated_act_and_mul- from freetoken.moe.cpu_executor import CpuMoeExecutor- from freetoken.third_party.gguf import GGMLQuantizationType-- torch.manual_seed(77)- L, E, H, I, top_k = 2, 16, 2816, 704, 8- layer, bs = 1, 4- dev = torch.device("cuda")- cache = _make_q4_0_cache(L, E, H, I)-- ex = CpuMoeExecutor(- cache, top_k=top_k, activation="gelu_tanh",- apply_router_weight_on_input=False, num_threads=0, max_tokens=bs, device=dev,- )-- hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16) * 0.5- ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)- w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)-- cpu_out = ex.decode(layer, hidden, w, ids).float()- torch.cuda.synchronize()-- b = cache.bank_sources- q4_0 = int(GGMLQuantizationType.Q4_0)- h = ggml.mul_mat_id(b["gate_up"][layer].to(dev), q4_0, hidden.float(), ids)- inter = torch.empty(bs, top_k, I, device=dev)- gated_act_and_mul("gelu_tanh", h.view(-1, 2 * I), inter.view(-1, I))- y = ggml.mul_mat_id(b["down"][layer].to(dev), q4_0, inter, ids)- gpu_out = ggml.moe_weighted_sum(y, w)-- rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)- assert rel < 6e-2, f"q4_0 vs ggml-mmvq rel err {rel.item()}"---def test_cpu_moe_decode_q4_0_cuda_graph_replay():- """Q4_0 CPU path under capture/replay: the host nodes must recompute the GEMV from- the freshly written pinned routing on each replay (dep flows through pinned buffers)."""- from freetoken.moe.cpu_executor import CpuMoeExecutor- from freetoken.moe.fused import fused_experts_decode_impl-- torch.manual_seed(9)- L, E, H, I, top_k = 2, 8, 2816, 704, 8- layer, bs = 1, 4- cache = _make_q4_0_cache(L, E, H, I)-- dev = torch.device("cuda")- stream = torch.cuda.Stream()- torch.cuda.set_stream(stream)-- ex = CpuMoeExecutor(- cache, top_k=top_k, activation="gelu_tanh",- apply_router_weight_on_input=False, num_threads=8, max_tokens=bs, device=dev,- )- b = cache.bank_sources- gate_up_layer = _dequant_bank(b["gate_up"][layer], H, dev)- down_layer = _dequant_bank(b["down"][layer], I, dev)-- def reference(hidden, ids, w):- return fused_experts_decode_impl(- hidden, gate_up_layer, down_layer, w, ids.clone(), "gelu_tanh", False- ).float()-- hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16) * 0.5- ids = torch.randint(0, E, (bs, top_k), device=dev, dtype=torch.int32)- w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)-- ex.decode(layer, hidden, w, ids) # eager warmup: materialize buffers + task- torch.cuda.synchronize()-- g = torch.cuda.CUDAGraph()- with torch.cuda.graph(g, stream=stream):- out_cap = ex.decode(layer, hidden, w, ids)- torch.cuda.synchronize()-- for it in range(3):- torch.manual_seed(600 + it)- hidden.copy_(torch.randn(bs, H, dtype=torch.bfloat16) * 0.5)- ids.copy_(torch.randint(0, E, (bs, top_k), dtype=torch.int32))- w.copy_(torch.rand(bs, top_k, dtype=torch.float32))- g.replay()- torch.cuda.synchronize()- ref = reference(hidden, ids, w)- rel = (out_cap.float() - ref).abs().max() / (ref.abs().max() + 1e-6)- assert rel < 2e-2, f"q4_0 replay {it} rel err {rel.item()}"-- print("cpu moe cuda graph replay (q4_0) OK")---if __name__ == "__main__":- for bs in (1, 3, 8):- test_cpu_decode_q4_0_matches_dequant_then_gpu(bs)- print(f"q4_0 bs={bs} OK")- test_cpu_decode_q4_0_matches_ggml_mmvq()- print("q4_0 vs ggml-mmvq OK")- test_cpu_moe_decode_q4_0_cuda_graph_replay()