Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Performance improvements on Arm for legacy and k-quants (#453)
Kawrakow293a5284c493
diff --git a/llama.cpp/ggml-common.h b/llama.cpp/ggml-common.hindex 43c7978a..b7a17ccf 100644--- a/llama.cpp/ggml-common.h+++ b/llama.cpp/ggml-common.h@@ -203,6 +203,18 @@ typedef struct {} block_q8_1;static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding");+//[kawrakow] Need these two for performance on Arm+typedef struct {+ ggml_half d[8];+ int8_t qs[4*QK8_1];+} block_q8_1_x4;+static_assert(sizeof(block_q8_1_x4) == 4*sizeof(block_q8_1), "wrong q8_1_x4 block size/padding");+typedef struct {+ ggml_half d[4];+ int8_t qs[4*QK8_0];+} block_q8_0_x4;+static_assert(sizeof(block_q8_0_x4) == 4*sizeof(block_q8_0), "wrong q8_0_x4 block size/padding");+//// Super-block quantization structures//@@ -313,10 +325,11 @@ typedef struct {static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");// This is only used for intermediate quantization and dot products+// [kawrakow] Note: I have switched the order of bsums and qs. This results in some performance gain on Armtypedef struct {float d; // delta- int8_t qs[QK_K]; // quantsint16_t bsums[QK_K/16]; // sum of quants in groups of 16+ int8_t qs[QK_K]; // quants} block_q8_K;static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");diff --git a/llama.cpp/ggml-quants.inc b/llama.cpp/ggml-quants.incindex e5a86a73..4d8be616 100644--- a/llama.cpp/ggml-quants.inc+++ b/llama.cpp/ggml-quants.inc@@ -873,7 +873,11 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k)block_q8_0 * restrict y = vy;#if defined(__ARM_NEON)+ // [kawrakow] When running on Arm, we change how the data is layed out for performance reasons+ block_q8_0_x4 * y4 = (block_q8_0_x4 *)vy;+ int nb4 = 4*(nb/4);for (int i = 0; i < nb; i++) {+ int i4 = i/4, ir = i%4;float32x4_t srcv [8];float32x4_t asrcv[8];float32x4_t amaxv[8];@@ -890,16 +894,29 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k)const float d = amax / ((1 << 7) - 1);const float id = d ? 1.0f/d : 0.0f;- y[i].d = GGML_FP32_TO_FP16(d);+ // [kawrakow] When running on Arm, we change how the data is layed out for performance reasons+ if (i < nb4) {+ y4[i4].d[ir] = GGML_FP32_TO_FP16(d);+ } else {+ y[i].d = GGML_FP32_TO_FP16(d);+ }for (int j = 0; j < 8; j++) {const float32x4_t v = vmulq_n_f32(srcv[j], id);const int32x4_t vi = vcvtnq_s32_f32(v);- y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);- y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);- y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);- y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);+ // [kawrakow] When running on Arm, we change how the data is layed out for performance reasons+ if (i < nb4) {+ y4[i4].qs[32*ir + 4*j + 0] = vgetq_lane_s32(vi, 0);+ y4[i4].qs[32*ir + 4*j + 1] = vgetq_lane_s32(vi, 1);+ y4[i4].qs[32*ir + 4*j + 2] = vgetq_lane_s32(vi, 2);+ y4[i4].qs[32*ir + 4*j + 3] = vgetq_lane_s32(vi, 3);+ } else {+ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);+ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);+ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);+ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);+ }}}#elif defined(__wasm_simd128__)@@ -1192,7 +1209,11 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k)block_q8_1 * restrict y = vy;#if defined(__ARM_NEON)+ // [kawrakow] When running on Arm, we change how the data is layed out for performance reasons+ block_q8_1_x4 * restrict y4 = vy;+ int nb4 = 4*(nb/4);for (int i = 0; i < nb; i++) {+ int i4 = i/4, ir = i%4;float32x4_t srcv [8];float32x4_t asrcv[8];float32x4_t amaxv[8];@@ -1209,7 +1230,12 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k)const float d = amax / ((1 << 7) - 1);const float id = d ? 1.0f/d : 0.0f;- y[i].d = GGML_FP32_TO_FP16(d);+ // [kawrakow] When running on Arm, we change how the data is layed out for performance reasons+ if (i < nb4) {+ y4[i4].d[ir] = GGML_FP32_TO_FP16(d);+ } else {+ y[i].d = GGML_FP32_TO_FP16(d);+ }int32x4_t accv = vdupq_n_s32(0);@@ -1217,15 +1243,28 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k)const float32x4_t v = vmulq_n_f32(srcv[j], id);const int32x4_t vi = vcvtnq_s32_f32(v);- y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);- y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);- y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);- y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);+ // [kawrakow] When running on Arm, we change how the data is layed out for performance reasons+ if (i < nb4) {+ y4[i4].qs[QK8_1*ir + 4*j + 0] = vgetq_lane_s32(vi, 0);+ y4[i4].qs[QK8_1*ir + 4*j + 1] = vgetq_lane_s32(vi, 1);+ y4[i4].qs[QK8_1*ir + 4*j + 2] = vgetq_lane_s32(vi, 2);+ y4[i4].qs[QK8_1*ir + 4*j + 3] = vgetq_lane_s32(vi, 3);+ } else {+ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);+ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);+ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);+ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);+ }accv = vaddq_s32(accv, vi);}- y[i].s = GGML_FP32_TO_FP16(d * vaddvq_s32(accv));+ // [kawrakow] When running on Arm, we change how the data is layed out for performance reasons+ if (i < nb4) {+ y4[i4].d[ir+4] = GGML_FP32_TO_FP16(d * vaddvq_s32(accv));+ } else {+ y[i].s = GGML_FP32_TO_FP16(d * vaddvq_s32(accv));+ }}#elif defined(__wasm_simd128__)for (int i = 0; i < nb; i++) {diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex 8659a942..fbefc079 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -65,10 +65,12 @@ static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrixstatic const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std::string & ftype_str_out) {- std::string ftype_str;+ std::string ftype_str; ftype_str.reserve(ftype_str_in.size());+ bool is_number = true;for (auto ch : ftype_str_in) {ftype_str.push_back(std::toupper(ch));+ if (!std::isdigit(ftype_str.back())) is_number = false;}for (auto & it : QUANT_OPTIONS) {if (it.name == ftype_str) {@@ -77,6 +79,9 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftypreturn true;}}+ // On my system (OS Ventura 13.2.1) calling std::stoi with invalid input leads to a crash (Segmentation fault 11)+ // Hence the check above and the early return+ if (!is_number) return false;try {int ftype_int = std::stoi(ftype_str);for (auto & it : QUANT_OPTIONS) {diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 95ee2ec1..30f4eab0 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -91,6 +91,7 @@ o/$(MODE)/llamafile: \o/$(MODE)/llamafile/sgemm.o: private CXXFLAGS += -Oso/$(MODE)/llamafile/iqk_mul_mat_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mavx2 -Xx86_64-mfma -Xx86_64-mf16co/$(MODE)/llamafile/iqk_mul_mat_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mavx2 -Xx86_64-mfma -Xx86_64-mf16c -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bw -Xx86_64-mavx512dq+o/$(MODE)/llamafile/iqk_mul_mat_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16co/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16co/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfmadiff --git a/llamafile/iqk_mul_mat.inc b/llamafile/iqk_mul_mat.incindex d41ed8b9..a126f2c9 100644--- a/llamafile/iqk_mul_mat.inc+++ b/llamafile/iqk_mul_mat.inc@@ -15,7 +15,8 @@// See the License for the specific language governing permissions and// limitations under the License.-#ifdef __x86_64__+#include <type_traits>+#if defined __x86_64__ || defined __aarch64__#include "llama.cpp/ggml-impl.h"#include "llama.cpp/ggml-quants.h"@@ -38,11 +39,6 @@#include <utility>#include <array>-#if defined HAVE_FANCY_SIMD- #undef HAVE_FANCY_SIMD-#endif-#if defined(__AVX512F__) && defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512BW__) && defined(__AVX512DQ__)- #define HAVE_FANCY_SIMD#endifnamespace {@@ -82,6 +78,40 @@ struct DataInfo {}};+typedef void (*mul_mat_t)(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x);++struct MulMat {+ std::array<mul_mat_t, 8> funcs = {};+ //std::array<mul_mat_t, 4> funcs = {};+ inline void mul_mat_NxM(int n, const void * vx, size_t bx, DataInfo& info, int nrc_x, int nrc_y) {+#ifdef __aarch64__+ constexpr int k_x_step = 64; //8192; // Tiling does not seem to help on my M2 Max (but difference to tiling is small)+#else+ constexpr int k_x_step = 64; // This works best on my Ryzen-7950X (but differences to other tile size are small)+#endif+ int n_step = (nrc_y - info.cur_y)/funcs.size();+ if (n_step > 0) {+ for (int ix = 0; ix < nrc_x; ix += k_x_step) {+ auto this_info = info;+ this_info.s += ix;+ int this_nrc_x = ix + k_x_step <= nrc_x ? k_x_step : nrc_x - ix;+ for (int iy = 0; iy < n_step; ++iy) {+ funcs.back()(n, (const void *)((const char *)vx + ix*bx), bx, this_info, this_nrc_x);+ this_info.cur_y += funcs.size();+ }+ }+ info.cur_y += funcs.size() * n_step;+ }+ int n_left = nrc_y - info.cur_y;+ if (n_left > 0) {+ funcs[n_left-1](n, vx, bx, info, nrc_x);+ }+ }+ static bool set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8, int Ny);+private:+ template <typename Dequantizer> static void set_functions(MulMat& m);+};+inline void make_q4_scales(const uint8_t * scales8, uint32_t * aux32) {const uint16_t * scales = (const uint16_t *)scales8;const uint32_t a0 = scales[0] | (scales[1] << 16);@@ -93,19 +123,71 @@ inline void make_q4_scales(const uint8_t * scales8, uint32_t * aux32) {aux32[0] = a0 & 0x3f3f3f3f;}-static inline float hsum_float_4(__m128 x) {+}++bool iqk_mul_mat(long Nx, long Ny, long ne00, int typeA, const void * A, const void * B,+ float * C, long stride_C, int ith, int nth) {++ MulMat mm;+ int row_size_q8;+ if (!MulMat::set_mul_mat(typeA, ne00, mm, row_size_q8, Ny)) {+ return false;+ }++ auto row_size_qx = ggml_row_size((ggml_type)typeA, ne00);++ auto nrc_x = (Nx + nth - 1)/nth;+ auto first_x = ith*nrc_x;+ if (first_x + nrc_x > Nx) nrc_x = Nx - first_x;++ DataInfo info{C + first_x, (const char *)B, (size_t)stride_C, (size_t)row_size_q8, 0, 1, nullptr, 0};++ mm.mul_mat_NxM(ne00, (const char *)A + row_size_qx*first_x, row_size_qx, info, nrc_x, Ny);++ return true;+}++bool iqk_mul_mat_moe(long Nx, long Ny, long ne00, int ne11, int typeA, const void * A, const void * B,+ float * C, long nb1, long nb2, const void * vrow_mapping, int ith, int nth) {+ const mmid_row_mapping * row_mapping = (const mmid_row_mapping *)vrow_mapping;+ assert(row_mapping != nullptr);++ MulMat mm;+ int row_size_q8;+ if (!MulMat::set_mul_mat(typeA, ne00, mm, row_size_q8, Ny)) {+ return false;+ }+ int row_size_qx = ggml_row_size((ggml_type)typeA, ne00);+ int nrc_x = (Nx + nth - 1)/nth;+ int first_x = ith*nrc_x;+ if (first_x + nrc_x > Nx) nrc_x = Nx - first_x;+ DataInfo info{C + first_x, (const char *)B, nb1/sizeof(float), (size_t)row_size_q8, 0, ne11, row_mapping, nb2/sizeof(float)};+ mm.mul_mat_NxM(ne00, (const char *)A + row_size_qx*first_x, row_size_qx, info, nrc_x, Ny);+ return true;+}++#if defined __x86_64__++#if defined HAVE_FANCY_SIMD+ #undef HAVE_FANCY_SIMD+#endif+#if defined(__AVX512F__) && defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512BW__) && defined(__AVX512DQ__)+ #define HAVE_FANCY_SIMD+#endif++namespace {++inline float hsum_float_4(__m128 x) {x = _mm_add_ps(x, _mm_movehl_ps(x, x));x = _mm_add_ss(x, _mm_movehdup_ps(x));return _mm_cvtss_f32(x);}-static inline float hsum_float_8(__m256 x) {+inline float hsum_float_8(__m256 x) {return hsum_float_4(_mm_add_ps(_mm256_castps256_ps128(x), _mm256_extractf128_ps(x, 1)));}#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)-typedef void (*mul_mat_t)(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x);-template <int nrc, typename block_q8 = block_q8_K> struct Q8 {@@ -748,7 +830,7 @@ struct DequantizerQ6K final : public BaseDequantizer<block_q6_K> {const __m256i mh = _mm256_set1_epi8(0x30);};-static inline __m256i get_scale_shuffle_16(int i) {+inline __m256i get_scale_shuffle_16(int i) {static const uint8_t k_shuffle[128] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,@@ -1212,29 +1294,7 @@ void mul_mat_q8_0_q8_0_T(int n, const void * vx, size_t bx, const DataInfo& info}}-struct MulMat {- std::array<mul_mat_t, 8> funcs = {};- inline void mul_mat_NxM(int n, const void * vx, size_t bx, DataInfo& info, int nrc_x, int nrc_y) {- constexpr int k_x_step = 64; // This works best on my Ryzen-7950X (but differences to other tile size are small)- int n_step = (nrc_y - info.cur_y)/funcs.size();- if (n_step > 0) {- for (int ix = 0; ix < nrc_x; ix += k_x_step) {- auto this_info = info;- this_info.s += ix;- int this_nrc_x = ix + k_x_step <= nrc_x ? k_x_step : nrc_x - ix;- for (int iy = 0; iy < n_step; ++iy) {- funcs.back()(n, (const void *)((const char *)vx + ix*bx), bx, this_info, this_nrc_x);- this_info.cur_y += funcs.size();- }- }- info.cur_y += funcs.size() * n_step;- }- int n_left = nrc_y - info.cur_y;- if (n_left > 0) {- funcs[n_left-1](n, vx, bx, info, nrc_x);- }- }- template <typename Dequantizer> static void set_functions(MulMat& m) {+template <typename Dequantizer> void MulMat::set_functions(MulMat& m) {if constexpr (std::is_same_v<Dequantizer, Q4_0_Unpacker> || std::is_same_v<Dequantizer, Q5_0_Unpacker>) {m.funcs[0] = mul_mat_qX_0_q8_0_T<Dequantizer, 1>;m.funcs[1] = mul_mat_qX_0_q8_0_T<Dequantizer, 2>;@@ -1289,10 +1349,9 @@ struct MulMat {}#endif}- }-};+}-bool set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8) {+bool MulMat::set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8, int) {row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00);@@ -1322,22 +1381,22 @@ bool set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8) {MulMat::set_functions<DequantizerIQ4XS>(mm);break;case GGML_TYPE_Q4_0:- assert (ne00 % Q4K_0 == 0);+ assert (ne00 % QK4_0 == 0);MulMat::set_functions<Q4_0_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);break;case GGML_TYPE_Q4_1:- assert (ne00 % Q4K_1 == 0);+ assert (ne00 % QK4_1 == 0);MulMat::set_functions<Q4_1_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);break;case GGML_TYPE_Q5_0:- assert (ne00 % Q5K_0 == 0);+ assert (ne00 % QK5_0 == 0);MulMat::set_functions<Q5_0_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);break;case GGML_TYPE_Q5_1:- assert (ne00 % Q5K_1 == 0);+ assert (ne00 % QK5_1 == 0);MulMat::set_functions<Q5_1_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);break;@@ -1351,49 +1410,1085 @@ bool set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8) {} // namespace-//-// ============================== Matrix multiplications-//-bool iqk_mul_mat(long Nx, long Ny, long ne00, int typeA, const void * A, const void * B,- float * C, long stride_C, int ith, int nth) {+#else // __aarch64__- MulMat mm;- int row_size_q8;- if (!set_mul_mat(typeA, ne00, mm, row_size_q8)) {- return false;+namespace {++template <int nrc, typename block_q8 = block_q8_K> struct Q8 {++ constexpr static int nrc_y = nrc;++ Q8(const DataInfo& info) {+ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8 *)info.src1_row(iy);}- auto row_size_qx = ggml_row_size((ggml_type)typeA, ne00);+ inline int8x16x2_t load_quants(int iy, int i, int j) const { return vld1q_s8_x2(y[iy][i].qs + 32*j); }+ inline int8x16x4_t load_quants_64(int iy, int i, int j) const { return vld1q_s8_x4(y[iy][i].qs + 64*j); }+ inline int16x8x2_t load_bsums(int iy, int i) const { return vld1q_s16_x2(y[iy][i].bsums); }+ inline int16x8_t load_bsums8(int iy, int i) const {+ auto q8s = vld1q_s16_x2(y[iy][i].bsums);+ return vpaddq_s16(q8s.val[0], q8s.val[1]);+ }+ inline float scale(int iy, int i) const { return y[iy][i].d; }- auto nrc_x = (Nx + nth - 1)/nth;- auto first_x = ith*nrc_x;- if (first_x + nrc_x > Nx) nrc_x = Nx - first_x;+ const block_q8 * y[nrc_y];+};- DataInfo info{C + first_x, (const char *)B, (size_t)stride_C, (size_t)row_size_q8, 0, 1, nullptr, 0};+template <typename Q8>+inline void compute_8_blocks(const uint8x16x4_t& qx_1, const uint8x16x4_t& qx_2, const Q8& q8,+ const int32x4x2_t& scales, int iy, int i, int j, int32x4_t& sumi) {+ auto mzero = vdupq_n_s32(0);+ auto q8b_1 = q8.load_quants(iy, i, 4*j+0);+ auto p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[0]), q8b_1.val[0]),+ vreinterpretq_s8_u8(qx_1.val[1]), q8b_1.val[1]); // block 1+ auto q8b_2 = q8.load_quants(iy, i, 4*j+1);+ auto p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[2]), q8b_2.val[0]),+ vreinterpretq_s8_u8(qx_1.val[3]), q8b_2.val[1]); // block 2+ auto p12 = vpaddq_s32(p1, p2);++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2);+ auto p3 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[0]), q8b_3.val[0]),+ vreinterpretq_s8_u8(qx_2.val[1]), q8b_3.val[1]); // block 1+ auto q8b_4 = q8.load_quants(iy, i, 4*j+3);+ auto p4 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[2]), q8b_4.val[0]),+ vreinterpretq_s8_u8(qx_2.val[3]), q8b_4.val[1]); // block 2+ auto p34 = vpaddq_s32(p3, p4);++ auto pall = vpaddq_s32(p12, p34);+ sumi = vmlaq_s32(sumi, scales.val[j], pall);+}- mm.mul_mat_NxM(ne00, (const char *)A + row_size_qx*first_x, row_size_qx, info, nrc_x, Ny);+template <typename Q8>+inline void compute_16_blocks(const uint8x16x4_t& qx_1, const uint8x16x4_t& qx_2, const Q8& q8,+ const int32x4x4_t& scales, int iy, int i, int j, int32x4_t& sumi) {++ auto mzero = vdupq_n_s32(0);+ auto q8b_1 = q8.load_quants(iy, i, 4*j+0);+ auto p1 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[0]), q8b_1.val[0]),+ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[1]), q8b_1.val[1])); // blocks 0, 0, 1, 1,+ auto q8b_2 = q8.load_quants(iy, i, 4*j+1);+ auto p2 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[2]), q8b_2.val[0]),+ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[3]), q8b_2.val[1])); // blocks 3, 3, 4, 4,+ auto p12 = vpaddq_s32(p1, p2); // blocks 0, 1, 2, 3+ sumi = vmlaq_s32(sumi, scales.val[2*j+0], p12);++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2);+ auto p3 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[0]), q8b_3.val[0]),+ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[1]), q8b_3.val[1])); // block 4, 4, 5, 5,+ auto q8b_4 = q8.load_quants(iy, i, 4*j+3);+ auto p4 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[2]), q8b_4.val[0]),+ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[3]), q8b_4.val[1])); // block 6, 6, 7, 7,+ auto p34 = vpaddq_s32(p3, p4); // blocks 4, 5, 6, 7+ sumi = vmlaq_s32(sumi, scales.val[2*j+1], p34);+}- return true;+template <typename Q8>+inline void accum_mins_8(const int16x8_t& mins, const Q8& q8, float32x4_t * acc, int i, float c) {+ for (int iy = 0; iy < Q8::nrc_y; ++iy) {+ auto q8s = q8.load_bsums8(iy, i);+ int32x4_t b1 = vmull_s16(vget_low_s16(mins), vget_low_s16(q8s));+ int32x4_t b2 = vmull_s16(vget_high_s16(mins), vget_high_s16(q8s));+ float32x4_t prod = vcvtq_f32_s32(vaddq_s32(b1, b2));+ acc[iy] = vmlaq_f32(acc[iy], prod, vdupq_n_f32(c*q8.scale(iy, i)));+ }+}+template <typename Q8>+inline void accum_mins_16(const int16x8x2_t& mins, const Q8& q8, float32x4_t * acc, int i, float c) {+ for (int iy = 0; iy < Q8::nrc_y; ++iy) {+ auto q8s = q8.load_bsums(iy, i);+ int32x4_t b1 = vmull_s16(vget_low_s16 (mins.val[0]), vget_low_s16 (q8s.val[0]));+ int32x4_t b2 = vmull_s16(vget_high_s16(mins.val[0]), vget_high_s16(q8s.val[0]));+ int32x4_t b3 = vmull_s16(vget_low_s16 (mins.val[1]), vget_low_s16 (q8s.val[1]));+ int32x4_t b4 = vmull_s16(vget_high_s16(mins.val[1]), vget_high_s16(q8s.val[1]));+ float32x4_t prod = vcvtq_f32_s32(vaddq_s32(vaddq_s32(b1, b2), vaddq_s32(b3, b4)));+ acc[iy] = vmlaq_f32(acc[iy], prod, vdupq_n_f32(c*q8.scale(iy, i)));+ }}-bool iqk_mul_mat_moe(long Nx, long Ny, long ne00, int ne11, int typeA, const void * A, const void * B,- float * C, long nb1, long nb2, const void * vrow_mapping, int ith, int nth) {- const mmid_row_mapping * row_mapping = (const mmid_row_mapping *)vrow_mapping;- assert(row_mapping != nullptr);+struct Scales8 {+ uint32_t utmp[4];+ const uint8_t * sc8 = (const uint8_t *)utmp;+ template <typename Q8, typename Qx>+ inline int32x4x2_t process_scales_mins(const Qx& x, const Q8& q8, int i, float32x4_t * acc) {+ make_q4_scales(x.scales, utmp);+ int16x8_t mins = vmovl_s8(vld1_s8((const int8_t *)sc8 + 8));+ accum_mins_8(mins, q8, acc, i, -GGML_FP16_TO_FP32(x.dmin));++ uint8x8_t scales8 = vld1_u8(sc8);+ uint16x8_t scales16 = vmovl_u8(scales8);+ int32x4x2_t scales = {vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales16))),+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales16)))};+ return scales;+ }+};- MulMat mm;- int row_size_q8;- if (!set_mul_mat(typeA, ne00, mm, row_size_q8)) {- return false;+struct Q4bits {+ const uint8x16_t m4b = vdupq_n_u8(0xf);+ uint8x16x4_t b1, b2;+ inline void prepare4(uint8x16x4_t& b, const uint8x16_t * val) const {+ b.val[0] = vandq_u8(val[0], m4b);+ b.val[2] = vshrq_n_u8(val[0], 4);+ b.val[1] = vandq_u8(val[1], m4b);+ b.val[3] = vshrq_n_u8(val[1], 4);+ }+ inline void prepare4_16(uint8x16x4_t& b, const uint8x16_t * val) const {+ b.val[0] = vandq_u8(val[0], m4b);+ b.val[1] = vshrq_n_u8(val[0], 4);+ b.val[2] = vandq_u8(val[1], m4b);+ b.val[3] = vshrq_n_u8(val[1], 4);+ }+ inline void prepare(const uint8_t * qs) {+ auto q4bits = vld1q_u8_x2(qs);+ prepare4(b1, q4bits.val);+ q4bits = vld1q_u8_x2(qs+32);+ prepare4(b2, q4bits.val);+ }+ inline void prepare_v2(const uint8_t * qs) {+ auto q4bits = vld1q_u8_x4(qs);+ prepare4(b1, q4bits.val+0);+ prepare4(b2, q4bits.val+2);+ }+ inline void prepare64(const uint8_t * qs) {+ auto q4bits = vld1q_u8_x4(qs);+ b1.val[0] = vandq_u8(q4bits.val[0], m4b);+ b1.val[1] = vandq_u8(q4bits.val[1], m4b);+ b1.val[2] = vandq_u8(q4bits.val[2], m4b);+ b1.val[3] = vandq_u8(q4bits.val[3], m4b);+ b2.val[0] = vshrq_n_u8(q4bits.val[0], 4);+ b2.val[1] = vshrq_n_u8(q4bits.val[1], 4);+ b2.val[2] = vshrq_n_u8(q4bits.val[2], 4);+ b2.val[3] = vshrq_n_u8(q4bits.val[3], 4);+ }+ inline void prepare16(const uint8_t * qs) {+ auto q4bits = vld1q_u8_x2(qs);+ prepare4_16(b1, q4bits.val);+ q4bits = vld1q_u8_x2(qs+32);+ prepare4_16(b2, q4bits.val);+ }+ inline void prepare16_v2(const uint8_t * qs) {+ auto q4bits = vld1q_u8_x4(qs);+ prepare4_16(b1, q4bits.val+0);+ prepare4_16(b2, q4bits.val+2);}- int row_size_qx = ggml_row_size((ggml_type)typeA, ne00);- int nrc_x = (Nx + nth - 1)/nth;- int first_x = ith*nrc_x;- if (first_x + nrc_x > Nx) nrc_x = Nx - first_x;- DataInfo info{C + first_x, (const char *)B, nb1/sizeof(float), (size_t)row_size_q8, 0, ne11, row_mapping, nb2/sizeof(float)};- mm.mul_mat_NxM(ne00, (const char *)A + row_size_qx*first_x, row_size_qx, info, nrc_x, Ny);- return true;+};++struct Q2bits {+ const uint8x16_t m4b = vdupq_n_u8(0x03);+ uint8x16x4_t b1, b2;+ inline void prepare(const uint8_t * qs) {+ auto q2bits = vld1q_u8_x2(qs);+ b1.val[0] = vandq_u8(q2bits.val[0], m4b);+ b1.val[1] = vandq_u8(q2bits.val[1], m4b);++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2);+ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2);+ b1.val[2] = vandq_u8(q2bits.val[0], m4b);+ b1.val[3] = vandq_u8(q2bits.val[1], m4b);++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2);+ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2);+ b2.val[0] = vandq_u8(q2bits.val[0], m4b);+ b2.val[1] = vandq_u8(q2bits.val[1], m4b);++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2);+ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2);+ b2.val[2] = vandq_u8(q2bits.val[0], m4b);+ b2.val[3] = vandq_u8(q2bits.val[1], m4b);+ }+};++template <typename block_q>+struct BaseDequantizer {+ BaseDequantizer(const void * vx, size_t bx, int nrc) : vx(vx), x(nullptr), bx(bx), nrc(nrc) {}+ inline void new_row(int ix) { x = (const block_q *)((const char *)vx + ix*bx); }+ const void * vx;+ const block_q * x;+ const size_t bx;+ const int nrc;+};++struct DequantizerQ4K final : public BaseDequantizer<block_q4_K> {+ DequantizerQ4K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {}++ constexpr static int num_blocks() { return 8; }+ constexpr static bool should_scale_quants() { return false; }++ template <typename Q8>+ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) {+ d = GGML_FP16_TO_FP32(x[i].d);+ return s8.process_scales_mins(x[i], q8, i, acc);+ }+ inline void prepare(int i, int j) {+ if (nrc == 1) bits.prepare_v2(x[i].qs+64*j);+ else bits.prepare(x[i].qs+64*j);+ }++ Q4bits bits;+ Scales8 s8;++ float d;+};++struct HighBit5 {+ const uint8x16_t mhb = vdupq_n_u8(0x10);+ uint8x16x2_t bits;+ inline void apply(uint8x16x4_t& b1, uint8x16x4_t& b2, bool do_shift) {+ b1.val[0] = vorrq_u8(b1.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 4), mhb));+ b1.val[1] = vorrq_u8(b1.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 4), mhb));+ b1.val[2] = vorrq_u8(b1.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 3), mhb));+ b1.val[3] = vorrq_u8(b1.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 3), mhb));++ b2.val[0] = vorrq_u8(b2.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 2), mhb));+ b2.val[1] = vorrq_u8(b2.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 2), mhb));+ b2.val[2] = vorrq_u8(b2.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 1), mhb));+ b2.val[3] = vorrq_u8(b2.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 1), mhb));++ if (do_shift) {+ bits.val[0] = vshrq_n_u8(bits.val[0], 4);+ bits.val[1] = vshrq_n_u8(bits.val[1], 4);+ }+ }+};++struct HighBit3 {+ const uint8x16_t mhb = vdupq_n_u8(0x04);+ uint8x16x2_t bits;+ inline void apply(uint8x16x4_t& b1, uint8x16x4_t& b2, bool do_shift) {+ b1.val[0] = vorrq_u8(b1.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 2), mhb));+ b1.val[1] = vorrq_u8(b1.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 2), mhb));+ b1.val[2] = vorrq_u8(b1.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 1), mhb));+ b1.val[3] = vorrq_u8(b1.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 1), mhb));++ b2.val[0] = vorrq_u8(b2.val[0], vandq_u8(bits.val[0], mhb));+ b2.val[1] = vorrq_u8(b2.val[1], vandq_u8(bits.val[1], mhb));+ b2.val[2] = vorrq_u8(b2.val[2], vandq_u8(vshrq_n_u8(bits.val[0], 1), mhb));+ b2.val[3] = vorrq_u8(b2.val[3], vandq_u8(vshrq_n_u8(bits.val[1], 1), mhb));++ if (do_shift) {+ bits.val[0] = vshrq_n_u8(bits.val[0], 4);+ bits.val[1] = vshrq_n_u8(bits.val[1], 4);+ }+ }+};++struct DequantizerQ5K final : public BaseDequantizer<block_q5_K> {+ DequantizerQ5K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {}++ constexpr static int num_blocks() { return 8; }+ constexpr static bool should_scale_quants() { return false; }++ template <typename Q8>+ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) {+ d = GGML_FP16_TO_FP32(x[i].d);+ h.bits = vld1q_u8_x2(x[i].qh);+ return s8.process_scales_mins(x[i], q8, i, acc);+ }+ inline void prepare(int i, int j) {+ if (nrc == 1) bits.prepare_v2(x[i].qs+64*j);+ else bits.prepare(x[i].qs+64*j);+ h.apply(bits.b1, bits.b2, j == 0);+ }++ Q4bits bits;+ HighBit5 h;+ Scales8 s8;++ uint8x16x2_t hbits;++ float d;+};++inline int32x4x4_t make_wider(const int16x8x2_t& scales16) {+ int32x4x4_t scales = {+ vmovl_s16(vget_low_s16 (scales16.val[0])),+ vmovl_s16(vget_high_s16(scales16.val[0])),+ vmovl_s16(vget_low_s16 (scales16.val[1])),+ vmovl_s16(vget_high_s16(scales16.val[1])),+ };+ return scales;+}++template <typename Q8>+inline int32x4x4_t process_scales_mins_16(const int8x16_t& scales8, const Q8& q8, float32x4_t * acc, int i, float c) {+ int16x8x2_t scales16;+ scales16.val[0] = vmovl_s8(vget_low_s8(scales8));+ scales16.val[1] = vmovl_s8(vget_high_s8(scales8));+ accum_mins_16(scales16, q8, acc, i, c);+ return make_wider(scales16);+}++struct DequantizerQ6K final : public BaseDequantizer<block_q6_K> {+ DequantizerQ6K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {}++ constexpr static int num_blocks() { return 16; }+ constexpr static bool should_scale_quants() { return false; }++ template <typename Q8>+ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) {+ d = GGML_FP16_TO_FP32(x[i].d);+ return process_scales_mins_16(vld1q_s8(x[i].scales), q8, acc, i, -32.f*d);+ }+ inline void prepare(int i, int j) {++ auto hbits = vld1q_u8_x2(x[i].qh + 32*j);++ bits.prepare64(x[i].ql+64*j);+ bits.b1.val[0] = vorrq_u8(bits.b1.val[0], vandq_u8(vshlq_n_u8(hbits.val[0], 4), mhb));+ bits.b1.val[1] = vorrq_u8(bits.b1.val[1], vandq_u8(vshlq_n_u8(hbits.val[1], 4), mhb));+ bits.b1.val[2] = vorrq_u8(bits.b1.val[2], vandq_u8(vshlq_n_u8(hbits.val[0], 2), mhb));+ bits.b1.val[3] = vorrq_u8(bits.b1.val[3], vandq_u8(vshlq_n_u8(hbits.val[1], 2), mhb));++ bits.b2.val[0] = vorrq_u8(bits.b2.val[0], vandq_u8(hbits.val[0], mhb));+ bits.b2.val[1] = vorrq_u8(bits.b2.val[1], vandq_u8(hbits.val[1], mhb));+ bits.b2.val[2] = vorrq_u8(bits.b2.val[2], vandq_u8(vshrq_n_u8(hbits.val[0], 2), mhb));+ bits.b2.val[3] = vorrq_u8(bits.b2.val[3], vandq_u8(vshrq_n_u8(hbits.val[1], 2), mhb));++ }++ Q4bits bits;++ const uint8x16_t mhb = vdupq_n_u8(0x30);++ float d;+};++struct DequantizerQ3K final : public BaseDequantizer<block_q3_K> {+ DequantizerQ3K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {}++ constexpr static int num_blocks() { return 16; }+ constexpr static bool should_scale_quants() { return false; }++ template <typename Q8>+ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) {+ d = GGML_FP16_TO_FP32(x[i].d);+ h.bits = vld1q_u8_x2(x[i].hmask);+ const uint16_t * sc16 = (const uint16_t *)x[i].scales;+ uint32_t aux0 = sc16[0] | (sc16[1] << 16);+ uint32_t aux1 = sc16[2] | (sc16[3] << 16);+ uint32_t aux2 = sc16[4] | (sc16[5] << 16);+ aux32[0] = (aux0 & 0x0f0f0f0f) | ((aux2 << 4) & 0x30303030);+ aux32[1] = (aux1 & 0x0f0f0f0f) | ((aux2 << 2) & 0x30303030);+ aux32[2] = ((aux0 >> 4) & 0x0f0f0f0f) | ((aux2 >> 0) & 0x30303030);+ aux32[3] = ((aux1 >> 4) & 0x0f0f0f0f) | ((aux2 >> 2) & 0x30303030);+ return process_scales_mins_16(vaddq_s8(vld1q_s8((const int8_t *)aux32), vdupq_n_s8(-32)), q8, acc, i, -4.f*d);+ }++ inline void prepare(int i, int j) {+ bits.prepare(x[i].qs+32*j);+ h.apply(bits.b1, bits.b2, j == 0);+ }++ uint32_t aux32[4];++ Q2bits bits;++ const uint8x16_t mhb = vdupq_n_u8(0x04);+ HighBit3 h;++ float d;+};++struct DequantizerQ2K final : public BaseDequantizer<block_q2_K> {+ DequantizerQ2K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {}++ constexpr static int num_blocks() { return 16; }+ constexpr static bool should_scale_quants() { return true; }++ template <typename Q8>+ inline void process_scales(int i, const Q8& q8, float32x4_t * acc) {+ d = GGML_FP16_TO_FP32(x[i].d);+ auto scales_and_mins = vld1q_u8(x[i].scales);+ auto mins8 = vreinterpretq_s8_u8(vshrq_n_u8(scales_and_mins, 4));+ int16x8x2_t scales16;+ scales16.val[0] = vmovl_s8(vget_low_s8(mins8));+ scales16.val[1] = vmovl_s8(vget_high_s8(mins8));+ accum_mins_16(scales16, q8, acc, i, -GGML_FP16_TO_FP32(x[i].dmin));++ scales8 = vandq_u8(scales_and_mins, vdupq_n_u8(0xf));+ }++ template <typename Q8>+ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) {+ process_scales(i, q8, acc);+ int16x8x2_t scales16;+ scales16.val[0] = vmovl_s8(vget_low_s8(vreinterpretq_s8_u8(scales8)));+ scales16.val[1] = vmovl_s8(vget_high_s8(vreinterpretq_s8_u8(scales8)));+ return make_wider(scales16);+ }++ template <typename Q8>+ inline void compute(const Q8& q8, int i, int j, int32x4_t * sumi) {+ auto m1 = vdupq_n_u8(1);+ auto shuffle = vdupq_n_u8(8*j);+ bits.b1.val[0] = vmulq_u8(bits.b1.val[0], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ bits.b1.val[1] = vmulq_u8(bits.b1.val[1], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ bits.b1.val[2] = vmulq_u8(bits.b1.val[2], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ bits.b1.val[3] = vmulq_u8(bits.b1.val[3], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ bits.b2.val[0] = vmulq_u8(bits.b2.val[0], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ bits.b2.val[1] = vmulq_u8(bits.b2.val[1], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ bits.b2.val[2] = vmulq_u8(bits.b2.val[2], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ bits.b2.val[3] = vmulq_u8(bits.b2.val[3], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1);+ for (int iy = 0; iy < Q8::nrc_y; ++iy) {+ auto q8b_1 = q8.load_quants(iy, i, 4*j+0);+ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b1.val[0]), q8b_1.val[0]),+ vreinterpretq_s8_u8(bits.b1.val[1]), q8b_1.val[1]);++ auto q8b_2 = q8.load_quants(iy, i, 4*j+1);+ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b1.val[2]), q8b_2.val[0]),+ vreinterpretq_s8_u8(bits.b1.val[3]), q8b_2.val[1]);++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2);+ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b2.val[0]), q8b_3.val[0]),+ vreinterpretq_s8_u8(bits.b2.val[1]), q8b_3.val[1]);++ auto q8b_4 = q8.load_quants(iy, i, 4*j+3);+ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b2.val[2]), q8b_4.val[0]),+ vreinterpretq_s8_u8(bits.b2.val[3]), q8b_4.val[1]);+ }+ }++ inline void prepare(int i, int j) {+ bits.prepare(x[i].qs+32*j);+ }++ uint32_t aux32[4];++ uint8x16_t scales8;++ Q2bits bits;++ float d;+};++struct DequantizerIQ4XS final : public BaseDequantizer<block_iq4_xs> {++ static int8x16_t load_values() {+ static const int8_t iq4nl_values[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};+ return vld1q_s8(iq4nl_values);+ }++ DequantizerIQ4XS(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc), values(load_values()) {}++ constexpr static int num_blocks() { return 8; }+ constexpr static bool should_scale_quants() { return false; }++ inline void new_row(int ix) { x = (const block_iq4_xs *)((const char *)vx + bx*ix); }++ template <typename Q8>+ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) {+ (void)q8;+ (void)acc;+ d = GGML_FP16_TO_FP32(x[i].d);+ const uint16_t scales_h = x[i].scales_h;+ const uint16_t * scales_l = (const uint16_t *)x[i].scales_l;+ aux32[0] = scales_l[0] | (scales_l[1] << 16);+ aux32[1] = aux32[0] >> 4;+ // scl is ordered as 0, 2, 4, 6, 1, 3, 5, 7+ uint8x8_t scl8 = vand_u8(vld1_u8((const uint8_t *)aux32), vdup_n_u8(0xf));+ uint16_t * aux16 = (uint16_t *)aux32;+ aux16[0] = scales_h << 4; aux16[1] = scales_h << 2; aux16[2] = scales_h; aux16[3] = scales_h >> 2;+ // sch is ordered as 0, 4, 1, 5, 2, 6, 3, 7+ uint8x8_t sch8 = vand_u8(vld1_u8((const uint8_t *)aux16), vdup_n_u8(0x30));+ int8x8_t scales8 = vadd_s8(vreinterpret_s8_u8(vorr_u8(scl8, vtbl1_u8(sch8, vreinterpret_u8_u32(hshuff)))), vdup_n_s8(-32));+ // shuffle 0, 2, 4, 6, 1, 3, 5, 7 -> 0, 1, 2, 3, 4, 5, 6, 7+ scales8 = vtbl1_s8(scales8, vreinterpret_s8_u32(hshuff));+ int16x8_t scales16 = vmovl_s8(scales8);+ int32x4x2_t scales = {vmovl_s16(vget_low_s16(scales16)), vmovl_s16(vget_high_s16(scales16))};+ return scales;+ }+ inline void prepare(int i, int j) {+ bits.prepare16(x[i].qs+64*j);+ //if (nrc == 1) {+ // bits.prepare16_v2(x[i].qs+64*j);+ //} else {+ // bits.prepare16(x[i].qs+64*j);+ //}+ for (int k = 0; k < 4; ++k) {+ bits.b1.val[k] = vreinterpretq_u8_s8(vqtbl1q_s8(values, bits.b1.val[k]));+ bits.b2.val[k] = vreinterpretq_u8_s8(vqtbl1q_s8(values, bits.b2.val[k]));+ }+ }++ Q4bits bits;+ const int8x16_t values;+ uint32_t aux32[2];++ constexpr static uint32x2_t hshuff = {0x05010400, 0x07030602};++ float d;+};++template <int nrc_y, typename Dequantizer>+static void mul_mat_qX_K_q8_K_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) {+ assert(n % QK_K == 0);+ const int nb = n / QK_K;++ Q8<nrc_y, block_q8_K> q8(info);++ Dequantizer deq(vx, bx, nrc_y);++ for (int ix = 0; ix < nrc_x; ++ix) {++ deq.new_row(ix);++ float32x4_t acc[nrc_y];+ for (int iy = 0; iy < nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f);++ for (int i = 0; i < nb; ++i) {++ int32x4_t sumi[nrc_y];+ for (int iy = 0; iy < nrc_y; ++iy) sumi[iy] = vdupq_n_s32(0);++ if constexpr (nrc_y > 1 && Dequantizer::should_scale_quants()) {+ deq.process_scales(i, q8, acc);+ deq.prepare(i, 0);+ deq.compute(q8, i, 0, sumi);+ deq.prepare(i, 1);+ deq.compute(q8, i, 1, sumi);+ } else {+ if constexpr (Dequantizer::num_blocks() == 8) {+ auto scales = deq.new_block(i, q8, acc);+ deq.prepare(i, 0);+ for (int iy = 0; iy < nrc_y; ++iy) compute_8_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 0, sumi[iy]);+ deq.prepare(i, 1);+ for (int iy = 0; iy < nrc_y; ++iy) compute_8_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 1, sumi[iy]);+ }+ else if constexpr (Dequantizer::num_blocks() == 16) {+ auto scales = deq.new_block(i, q8, acc);+ deq.prepare(i, 0);+ for (int iy = 0; iy < nrc_y; ++iy) compute_16_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 0, sumi[iy]);+ deq.prepare(i, 1);+ for (int iy = 0; iy < nrc_y; ++iy) compute_16_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 1, sumi[iy]);+ }+ else {+ GGML_ASSERT(false);+ }+ }++ for (int iy = 0; iy < nrc_y; ++iy) {+ acc[iy] = vmlaq_f32(acc[iy], vcvtq_f32_s32(sumi[iy]), vdupq_n_f32(deq.d*q8.scale(iy, i)));+ }+ }++ for (int iy = 0; iy < nrc_y; ++iy) {+ info.store(ix, iy, vaddvq_f32(acc[iy]));+ }+ }+}++// =========================================== Legacy quants++template <typename Block>+inline float16x4_t load_scales_q0(const Block * x, ggml_half * aux) {+ for (int k = 0; k < 4; ++k) aux[k] = x[k].d;+ return vld1_f16((const float16_t *)aux);+}++template <typename Block>+inline float16x8_t load_scales_q1(const Block * x, ggml_half * aux) {+ if constexpr (std::is_same_v<Block, block_q8_1>) {+ for (int k = 0; k < 4; ++k) { aux[k] = x[k].d; aux[k+4] = x[k].s; }+ } else {+ for (int k = 0; k < 4; ++k) { aux[k] = x[k].d; aux[k+4] = x[k].m; }+ }+ return vld1q_f16((const float16_t *)aux);+}++struct Q4LegacyBits {+ template <typename Block>+ inline void prepare(const Block * x) {+ for (int i = 0; i < 4; ++i) {+ auto q4bits = vld1q_u8(x[i].qs);+ b[2*i+0] = vreinterpretq_s8_u8(vandq_u8(q4bits, m4b));+ b[2*i+1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits, 4));+ }+ }+ inline void prepare1(const uint8_t * qs, int8x16_t * q) const {+ auto q4bits = vld1q_u8(qs);+ q[0] = vreinterpretq_s8_u8(vandq_u8(q4bits, m4b));+ q[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits, 4));+ }+ inline void prepare1(const uint8_t * qs) {+ prepare1(qs, b);+ }+ const uint8x16_t m4b = vdupq_n_u8(0xf);+ int8x16_t b[8];+};++// One would think this commented out version would do better than the one below+// because it offers more opportunities to execute instructions in parallel.+// Instead, it runs significantly slower. Why? If the compiler is running out of vector registers+// cannot it just do the sequential version below on its own?+//inline int32x4_t sum_4_blocks(const int8x16_t * b, const int8_t * qs) {+// const auto q8b_1 = vld1q_s8_x2(qs + 0);+// auto p12 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[0], q8b_1.val[0]), b[1], q8b_1.val[1]);+// const auto q8b_2 = vld1q_s8_x2(qs + 32);+// auto p34 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[2], q8b_2.val[0]), b[3], q8b_2.val[1]);+// auto p1234 = vpaddq_s32(p12, p34);+// const auto q8b_3 = vld1q_s8_x2(qs + 64);+// auto p56 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[4], q8b_3.val[0]), b[5], q8b_3.val[1]);+// const auto q8b_4 = vld1q_s8_x2(qs + 96);+// auto p78 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[6], q8b_4.val[0]), b[7], q8b_4.val[1]);+// return vpaddq_s32(p1234, vpaddq_s32(p56, p78));+//}++inline int32x4_t sum_4_blocks(const int8x16_t * b, const int8_t * qs) {+ auto q8b = vld1q_s8_x2(qs + 0);+ auto p12 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[0], q8b.val[0]), b[1], q8b.val[1]);+ q8b = vld1q_s8_x2(qs + 32);+ auto p34 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[2], q8b.val[0]), b[3], q8b.val[1]);+ auto p1234 = vpaddq_s32(p12, p34);+ q8b = vld1q_s8_x2(qs + 64);+ auto p56 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[4], q8b.val[0]), b[5], q8b.val[1]);+ q8b = vld1q_s8_x2(qs + 96);+ auto p78 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[6], q8b.val[0]), b[7], q8b.val[1]);+ return vpaddq_s32(p1234, vpaddq_s32(p56, p78));+}++template <int nrc> struct Q80 {++ constexpr static int nrc_y = nrc;++ Q80(const DataInfo& info) {+ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8_0 *)info.src1_row(iy);+ }++ inline const int8_t * quant_data(int iy, int i) const {+ const block_q8_0_x4 * y4 = (const block_q8_0_x4 *)y[iy] + i;+ return y4->qs;+ }++ inline float16x4_t load_scales(int iy, int i) const {+ const block_q8_0_x4 * y4 = (const block_q8_0_x4 *)y[iy] + i;+ return vld1_f16((const float16_t *)y4->d);+ }++ template <typename Dequantizer>+ inline void process_scales(int i, Dequantizer& deq, float16x4_t * sc16, float32x4_t * /*acc*/) const {+ auto qx_scales = deq.new_block(i);+ for (int iy = 0; iy < nrc; ++iy) {+ auto q8_scales = load_scales(iy, i);+ sc16[iy] = vmul_f16(qx_scales, q8_scales);+ }+ }++ template <typename Dequantizer>+ inline void process_1_block(int i, Dequantizer& deq, float32x4_t * acc) const {+ deq.prepare1(i);+ float d = GGML_FP16_TO_FP32(deq.x[i].d);+ for (int iy = 0; iy < nrc; ++iy) {+ auto q8b = vld1q_s8_x2(y[iy][i].qs);+ auto p = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b[0], q8b.val[0]), deq.bits.b[1], q8b.val[1]);+ acc[iy] = vmlaq_f32(acc[iy], vdupq_n_f32(d*GGML_FP16_TO_FP32(y[iy][i].d)), vcvtq_f32_s32(p));+ }+ }++ const block_q8_0 * y[nrc_y];+};++template <int nrc> struct Q81 {++ constexpr static int nrc_y = nrc;++ Q81(const DataInfo& info) {+ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8_1 *)info.src1_row(iy);+ }++ inline const int8_t * quant_data(int iy, int i) const {+ const block_q8_1_x4 * y4 = (const block_q8_1_x4 *)y[iy] + i;+ return y4->qs;+ }++ inline float16x8_t load_scales(int iy, int i) const {+ const block_q8_1_x4 * y4 = (const block_q8_1_x4 *)y[iy] + i;+ return vld1q_f16((const float16_t *)y4->d);+ }++ template <typename Dequantizer>+ inline void process_scales(int i, Dequantizer& deq, float16x4_t * sc16, float32x4_t * acc) const {+ auto qx_scales = deq.new_block(i);+ for (int iy = 0; iy < nrc; ++iy) {+ auto q8_scales = load_scales(iy, i);+ auto m = vmul_f16(vget_high_f16(qx_scales), vget_high_f16(q8_scales));+ acc[iy] = vaddq_f32(acc[iy], vcvt_f32_f16(m));+ sc16[iy] = vmul_f16(vget_low_f16(qx_scales), vget_low_f16(q8_scales));+ }+ }++ template <typename Dequantizer>+ inline void process_1_block(int i, Dequantizer& deq, float32x4_t * acc) const {+ deq.prepare1(i);+ float d = GGML_FP16_TO_FP32(deq.x[i].d), m = 0.25f*GGML_FP16_TO_FP32(deq.x[i].m);+ for (int iy = 0; iy < nrc; ++iy) {+ auto q8b = vld1q_s8_x2(y[iy][i].qs);+ auto p = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b[0], q8b.val[0]), deq.bits.b[1], q8b.val[1]);+ acc[iy] = vmlaq_f32(acc[iy], vdupq_n_f32(d*GGML_FP16_TO_FP32(y[iy][i].d)), vcvtq_f32_s32(p));+ acc[iy] = vaddq_f32(acc[iy], vdupq_n_f32(m*GGML_FP16_TO_FP32(y[iy][i].s)));+ }+ }++ const block_q8_1 * y[nrc_y];+};++template <typename block_q>+struct BaseLegacyDequantizer {++ BaseLegacyDequantizer(const void * vx, size_t bx) : vx(vx), x(nullptr), bx(bx) {}++ inline void new_row(int ix) { x = (const block_q *)((const char *)vx + bx*ix); }++ Q4LegacyBits bits;++ const void * vx;+ const block_q * x;+ size_t bx;+};++struct DequantizerQ40 final : public BaseLegacyDequantizer<block_q4_0> {++ DequantizerQ40(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {}++ inline void prepare1(int i, int8x16_t * q) const {+ bits.prepare1(x[i].qs, q);+ q[0] = vaddq_s8(q[0], m8);+ q[1] = vaddq_s8(q[1], m8);+ }+ inline void prepare1(int i) {+ prepare1(i, bits.b);+ }++ inline float16x4_t new_block(int i) {+ ggml_half aux[4];+ for (int k = 0; k < 4; ++k) {+ aux[k] = x[4*i+k].d;+ prepare1(4*i+k, bits.b + 2*k);+ }+ return vld1_f16((const float16_t *)aux);+ }++ const int8x16_t m8 = vdupq_n_s8(-8);+ //ggml_half aux[4];+};++struct DequantizerQ41 : public BaseLegacyDequantizer<block_q4_1> {++ DequantizerQ41(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {}++ inline void prepare1(int i) {+ bits.prepare1(x[i].qs);+ }++ inline float16x8_t new_block(int i) {+ uint32_t aux32[4];+ const uint32_t * s32 = (const uint32_t *)&x[4*i].d;+ for (int k = 0; k < 4; ++k) {+ aux32[k] = *s32; s32 += sizeof(block_q4_1)/4;+ bits.prepare1(x[4*i+k].qs, bits.b + 2*k);+ }+ return vreinterpretq_f16_u8(vqtbl1q_u8(vld1q_u8((const uint8_t *)aux32), vreinterpretq_u8_u64(shuffle)));+ }+ // Leaving this commented out attempt to be reminded that I already tried this.+ // It has basically the same performance as the version above.+ //inline float16x8_t new_block(int i) {+ // uint32x4_t scales = {};+ // const block_q4_1 * xi = x + 4*i;+ // const uint32_t * s32 = (const uint32_t *)&xi->d;+ // scales = vsetq_lane_u32(*s32, scales, 0); s32 += sizeof(block_q4_1)/4;+ // bits.prepare1(xi[0].qs, bits.b + 0);+ // scales = vsetq_lane_u32(*s32, scales, 1); s32 += sizeof(block_q4_1)/4;+ // bits.prepare1(xi[1].qs, bits.b + 2);+ // scales = vsetq_lane_u32(*s32, scales, 2); s32 += sizeof(block_q4_1)/4;+ // bits.prepare1(xi[2].qs, bits.b + 4);+ // scales = vsetq_lane_u32(*s32, scales, 3);+ // bits.prepare1(xi[3].qs, bits.b + 6);+ // return vreinterpretq_f16_u8(vqtbl1q_u8(vreinterpretq_u8_u32(scales), vreinterpretq_u8_u64(shuffle)));+ //}++ const uint64x2_t shuffle = {0x0d0c090805040100, 0x0f0e0b0a07060302};+};++struct HighBit5Legacy {+ inline uint8x16_t to_bytes(const uint8_t * qh) const {+ uint8x16_t h = vqtbl1q_u8(vreinterpretq_u8_u16(vdupq_n_u16(*(const uint16_t *)qh)), shuffle);+ return vceqq_u8(vandq_u8(h, vreinterpretq_u8_u64(mask)), vreinterpretq_u8_u64(mask));+ }+ inline uint8x16_t to_negated_bytes(const uint8_t * qh) const {+ uint8x16_t h = vqtbl1q_u8(vreinterpretq_u8_u16(vdupq_n_u16(*(const uint16_t *)qh)), shuffle);+ return vceqq_u8(vandq_u8(h, vreinterpretq_u8_u64(mask)), vdupq_n_u8(0));+ }+ const uint64x2_t mask = vdupq_n_u64(0x8040201008040201);+ const uint8x16_t shuffle = vcombine_u8(vdup_n_u8(0), vdup_n_u8(1));+};++struct DequantizerQ50 final : public BaseLegacyDequantizer<block_q5_0> {++ DequantizerQ50(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {}++ inline void prepare1(int i, int8x16_t * q) const {+ bits.prepare1(x[i].qs, q);+ auto qh = x[i].qh;+ q[0] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[0]), vandq_u8(mh, hbits.to_negated_bytes(qh+0))));+ q[1] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[1]), vandq_u8(mh, hbits.to_negated_bytes(qh+2))));+ }+ inline void prepare1(int i) {+ prepare1(i, bits.b);+ }++ inline float16x4_t new_block(int i) {+ ggml_half aux[4];+ for (int k = 0; k < 4; ++k) {+ aux[k] = x[4*i+k].d;+ prepare1(4*i+k, bits.b + 2*k);+ }+ return vld1_f16((const float16_t *)aux);+ }++ HighBit5Legacy hbits;++ const uint8x16_t mh = vdupq_n_u8(0xf0);++};++struct DequantizerQ80 final : public BaseLegacyDequantizer<block_q8_0> {++ DequantizerQ80(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {}++ inline void prepare1(int i) {+ bits.b[0] = vld1q_s8(x[i].qs);+ bits.b[1] = vld1q_s8(x[i].qs+16);+ }++ inline float16x4_t new_block(int i) {+ ggml_half aux[4];+ for (int k = 0; k < 4; ++k) {+ aux[k] = x[4*i+k].d;+ bits.b[2*k+0] = vld1q_s8(x[4*i+k].qs);+ bits.b[2*k+1] = vld1q_s8(x[4*i+k].qs+16);+ }+ return vld1_f16((const float16_t *)aux);+ }++};++struct DequantizerQ51 final : public BaseLegacyDequantizer<block_q5_1> {++ DequantizerQ51(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {}++ inline void prepare1(int i, int8x16_t * q) const {+ bits.prepare1(x[i].qs, q);+ auto qh = x[i].qh;+ q[0] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[0]), vandq_u8(mh, hbits.to_bytes(qh+0))));+ q[1] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[1]), vandq_u8(mh, hbits.to_bytes(qh+2))));+ }+ inline void prepare1(int i) {+ bits.prepare1(x[i].qs, bits.b);+ }++ inline float16x8_t new_block(int i) {+ uint32_t aux32[4];+ const uint32_t * s32 = (const uint32_t *)&x[4*i].d;+ for (int k = 0; k < 4; ++k) {+ aux32[k] = *s32; s32 += sizeof(block_q5_1)/4;+ prepare1(4*i+k, bits.b + 2*k);+ }+ return vreinterpretq_f16_u8(vqtbl1q_u8(vld1q_u8((const uint8_t *)aux32), vreinterpretq_u8_u64(shuffle)));+ }++ HighBit5Legacy hbits;++ const uint8x16_t mh = vdupq_n_u8(0x10);+ const uint64x2_t shuffle = {0x0d0c090805040100, 0x0f0e0b0a07060302};++};++template <typename Dequantizer, typename Q8>+inline void sum_4(int i, Dequantizer& deq, const Q8& q8, const float16x4_t * sc16, float32x4_t * acc) {+ for (int iy = 0; iy < Q8::nrc_y; ++iy) {+ auto pall = sum_4_blocks(deq.bits.b, q8.quant_data(iy, i));+ auto scale = vcvt_f32_f16(sc16[iy]);+ acc[iy] = vmlaq_f32(acc[iy], scale, vcvtq_f32_s32(pall));+ }+}++template <typename Dequantizer, typename Q8>+inline void mul_mat_qX_Y_q8_Y(int n, Dequantizer& deq, Q8& q8, const DataInfo& info, int nrc_x) {+ const int nb = n / QK4_1;++ float16x4_t sc16[Q8::nrc_y];++ for (int ix = 0; ix < nrc_x; ++ix) {++ deq.new_row(ix);++ float32x4_t acc[Q8::nrc_y];+ for (int iy = 0; iy < Q8::nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f);++ for (int i = 0; i < nb/4; ++i) {+ q8.process_scales(i, deq, sc16, acc);+ sum_4(i, deq, q8, sc16, acc);+ }+ for (int i = 4*(nb/4); i < nb; ++i) {+ q8.process_1_block(i, deq, acc);+ }++ for (int iy = 0; iy < Q8::nrc_y; ++iy) {+ info.store(ix, iy, vaddvq_f32(acc[iy]));+ }+ }+}++template <typename Dequantizer, typename Q8>+inline void mul_mat_qX_Y_q8_Y_1(int n, Dequantizer& deq1, Dequantizer& deq2, Q8& q8, const DataInfo& info, int nrc_x) {+ const int nb = n / QK4_1;++ float16x4_t sc16[2];++ for (int ix = 0; ix < nrc_x; ++ix) {++ deq1.new_row(ix);+ deq2.new_row(ix);++ float32x4_t acc[2] = { vdupq_n_f32(0.f), vdupq_n_f32(0.f) };++ for (int i = 0; i < nb/8; ++i) {+ q8.process_scales(2*i+0, deq1, sc16+0, acc+0);+ q8.process_scales(2*i+1, deq2, sc16+1, acc+1);+ sum_4(2*i+0, deq1, q8, sc16+0, acc+0);+ sum_4(2*i+1, deq2, q8, sc16+1, acc+1);+ }+ for (int i = 2*(nb/8); i < nb/4; ++i) {+ q8.process_scales(i, deq1, sc16, acc);+ sum_4(i, deq1, q8, sc16, acc);+ }+ for (int i = 4*(nb/4); i < nb; ++i) {+ q8.process_1_block(i, deq1, acc);+ }++ info.store(ix, 0, vaddvq_f32(vaddq_f32(acc[0], acc[1])));+ }+}++template <typename Dequantizer, int nrc_y>+static void mul_mat_qX_1_q8_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) {+ Q81<nrc_y> q8(info);+ if constexpr (nrc_y == 1) {+ Dequantizer deq1(vx, bx), deq2(vx, bx);+ mul_mat_qX_Y_q8_Y_1(n, deq1, deq2, q8, info, nrc_x);+ } else {+ Dequantizer deq(vx, bx);+ mul_mat_qX_Y_q8_Y(n, deq, q8, info, nrc_x);+ }+}++template <typename Dequantizer, int nrc_y>+static void mul_mat_qX_0_q8_0(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) {+ Q80<nrc_y> q8(info);+ if constexpr (nrc_y == 1) {+ Dequantizer deq1(vx, bx), deq2(vx, bx);+ mul_mat_qX_Y_q8_Y_1(n, deq1, deq2, q8, info, nrc_x);+ } else {+ Dequantizer deq(vx, bx);+ mul_mat_qX_Y_q8_Y(n, deq, q8, info, nrc_x);+ }+}++template <typename Dequantizer>+static void mul_mat_qX_1_q8_1_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) {+ Dequantizer deq1(vx, bx), deq2(vx, bx);+ Q81<1> q8(info);+ mul_mat_qX_Y_q8_Y_1(n, deq1, deq2, q8, info, nrc_x);+}++template <typename Dequantizer>+static void mul_mat_qX_0_q8_0_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) {+ Dequantizer deq1(vx, bx), deq2(vx, bx);+ Q80<1> q8(info);+ mul_mat_qX_Y_q8_Y(n, deq1, deq2, q8, info, nrc_x);+}++template <typename Dequantizer> void MulMat::set_functions(MulMat& m) {+ if constexpr (std::is_same_v<Dequantizer, DequantizerQ40> || std::is_same_v<Dequantizer, DequantizerQ50> ||+ std::is_same_v<Dequantizer, DequantizerQ80>) {+ m.funcs[0] = mul_mat_qX_0_q8_0<Dequantizer, 1>;+ m.funcs[1] = mul_mat_qX_0_q8_0<Dequantizer, 2>;+ m.funcs[2] = mul_mat_qX_0_q8_0<Dequantizer, 3>;+ m.funcs[3] = mul_mat_qX_0_q8_0<Dequantizer, 4>;+ m.funcs[4] = mul_mat_qX_0_q8_0<Dequantizer, 5>;+ m.funcs[5] = mul_mat_qX_0_q8_0<Dequantizer, 6>;+ m.funcs[6] = mul_mat_qX_0_q8_0<Dequantizer, 7>;+ m.funcs[7] = mul_mat_qX_0_q8_0<Dequantizer, 8>;+ }+ else if constexpr (std::is_same_v<Dequantizer, DequantizerQ41> || std::is_same_v<Dequantizer, DequantizerQ51>) {+ m.funcs[0] = mul_mat_qX_1_q8_1<Dequantizer, 1>;+ m.funcs[1] = mul_mat_qX_1_q8_1<Dequantizer, 2>;+ m.funcs[2] = mul_mat_qX_1_q8_1<Dequantizer, 3>;+ m.funcs[3] = mul_mat_qX_1_q8_1<Dequantizer, 4>;+ m.funcs[4] = mul_mat_qX_1_q8_1<Dequantizer, 5>;+ m.funcs[5] = mul_mat_qX_1_q8_1<Dequantizer, 6>;+ m.funcs[6] = mul_mat_qX_1_q8_1<Dequantizer, 7>;+ m.funcs[7] = mul_mat_qX_1_q8_1<Dequantizer, 8>;+ }+ else {+ m.funcs[0] = mul_mat_qX_K_q8_K_T<1, Dequantizer>;+ m.funcs[1] = mul_mat_qX_K_q8_K_T<2, Dequantizer>;+ m.funcs[2] = mul_mat_qX_K_q8_K_T<3, Dequantizer>;+ m.funcs[3] = mul_mat_qX_K_q8_K_T<4, Dequantizer>;+ m.funcs[4] = mul_mat_qX_K_q8_K_T<5, Dequantizer>;+ m.funcs[5] = mul_mat_qX_K_q8_K_T<6, Dequantizer>;+ m.funcs[6] = mul_mat_qX_K_q8_K_T<7, Dequantizer>;+ m.funcs[7] = mul_mat_qX_K_q8_K_T<8, Dequantizer>;+ }+}++bool MulMat::set_mul_mat(int typeA, int ne00, MulMat& m, int& row_size_q8, int /*Ny*/) {+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00);++ switch (typeA) {+ case GGML_TYPE_Q2_K:+ MulMat::set_functions<DequantizerQ2K>(m);+ break;+ case GGML_TYPE_Q3_K:+ MulMat::set_functions<DequantizerQ3K>(m);+ break;+ case GGML_TYPE_Q4_K:+ MulMat::set_functions<DequantizerQ4K>(m);+ break;+ case GGML_TYPE_Q5_K:+ MulMat::set_functions<DequantizerQ5K>(m);+ break;+ case GGML_TYPE_Q6_K:+ MulMat::set_functions<DequantizerQ6K>(m);+ break;+ case GGML_TYPE_IQ4_XS:+ MulMat::set_functions<DequantizerIQ4XS>(m);+ break;+ case GGML_TYPE_Q4_0:+ MulMat::set_functions<DequantizerQ40>(m);+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);+ break;+ case GGML_TYPE_Q4_1:+ MulMat::set_functions<DequantizerQ41>(m);+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);+ break;+ case GGML_TYPE_Q5_0:+ MulMat::set_functions<DequantizerQ50>(m);+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);+ break;+ case GGML_TYPE_Q5_1:+ MulMat::set_functions<DequantizerQ51>(m);+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);+ break;+ case GGML_TYPE_Q8_0:+ MulMat::set_functions<DequantizerQ80>(m);+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);+ break;+ default:+ return false;+ }+ return true;+}+}-#endif // __x86_64__+#endif // __x86_64__ or __aarch64__diff --git a/llamafile/iqk_mul_mat_arm82.cpp b/llamafile/iqk_mul_mat_arm82.cppnew file mode 100644index 00000000..592eb205--- /dev/null+++ b/llamafile/iqk_mul_mat_arm82.cpp@@ -0,0 +1,5 @@+#ifdef __aarch64__+#define iqk_mul_mat iqk_mul_mat_arm82+#define iqk_mul_mat_moe iqk_mul_mat_moe_arm82+#include "iqk_mul_mat.inc"+#endif // __aarch64__diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex d7324c1f..648e751f 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -88,6 +88,7 @@ static const struct GemmFuncs {// e.g. Apple M1, Raspberry Pi 5sgemm = llamafile_sgemm_arm82;mixmul = llamafile_mixmul_arm82;+ iqk_mixmul = iqk_mul_mat_moe_arm82;} else {// ARM64 baseline ISAsgemm = llamafile_sgemm_arm80;diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex 9e1864c8..33fcd603 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -9,11 +9,14 @@ struct ggml_compute_params;bool iqk_mul_mat(long, long, long, int, const void *, const void *, float *, long, int, int);bool iqk_mul_mat_zen4(long, long, long, int, const void *, const void *, float *, long, int, int);+bool iqk_mul_mat_arm82(long, long, long, int, const void *, const void *, float *, long, int, int);bool iqk_mul_mat_moe(long, long, long, int, int, const void *, const void *,float *, long, long, const void *, int, int);bool iqk_mul_mat_moe_zen4(long, long, long, int, int, const void *, const void *,float *, long, long, const void *, int, int);+bool iqk_mul_mat_moe_arm82(long, long, long, int, int, const void *, const void *,+ float *, long, long, const void *, int, int);bool iqk_mul_mat_moe_unsupported(long, long, long, int, int, const void *, const void *,float *, long, long, const void *, int, int);diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.incindex 7555e799..db01abc4 100644--- a/llamafile/tinyblas_cpu_sgemm.inc+++ b/llamafile/tinyblas_cpu_sgemm.inc@@ -322,7 +322,8 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidassert(nth > 0);assert(ith < nth);-#if defined(__x86_64__) && QK_K == 256+#if QK_K == 256+#if defined(__x86_64__)if (X86_CHECK(AVX2) && X86_CHECK(FMA)) {if (Btype == GGML_TYPE_Q8_K && Ctype == GGML_TYPE_F32) {if (iqk_mul_mat(m, n, k * QK_K, Atype, A, B, (float *)C, ldc, ith, nth)) {@@ -336,6 +337,19 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void}}}+#elif defined __aarch64__ && defined __ARM_FEATURE_DOTPROD && !defined _MSC_VER+ if (Btype == GGML_TYPE_Q8_K && Ctype == GGML_TYPE_F32) {+ if (iqk_mul_mat(m, n, k * QK_K, Atype, A, B, (float *)C, ldc, ith, nth)) {+ return true;+ }+ }+ if ((Btype == GGML_TYPE_Q8_0 || Btype == GGML_TYPE_Q8_1) && Ctype == GGML_TYPE_F32) {+ assert(QK8_0 == QK8_1 == QK4_0 == QK4_1 == QK5_0 == QK5_1 == 32);+ if (iqk_mul_mat(m, n, k * QK8_0, Atype, A, B, (float *)C, ldc, ith, nth)) {+ return true;+ }+ }+#endif#endifswitch (Ctype) {diff --git a/llamafile/tinyblas_cpu_sgemm_arm82.cpp b/llamafile/tinyblas_cpu_sgemm_arm82.cppindex 81e77ce2..5cfe23a4 100644--- a/llamafile/tinyblas_cpu_sgemm_arm82.cpp+++ b/llamafile/tinyblas_cpu_sgemm_arm82.cpp@@ -1,4 +1,5 @@#ifdef __aarch64__#define llamafile_sgemm llamafile_sgemm_arm82+#define iqk_mul_mat iqk_mul_mat_arm82#include "tinyblas_cpu_sgemm.inc"#endif // __aarch64__