LLM inference in C/C++
C++ ggml
kleidiai: add SME2 Q4_K and Q6_K matmul support (#29219)
Charles Xu45451c366d42
diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txtindex 50341fd6ab..d347757e18 100644--- a/ggml/src/ggml-cpu/CMakeLists.txt+++ b/ggml/src/ggml-cpu/CMakeLists.txt@@ -650,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name)# Fetch KleidiAI sources:include(FetchContent)- set(KLEIDIAI_COMMIT_TAG "v1.24.0")+ set(KLEIDIAI_COMMIT_TAG "v1.31.0")set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz")- set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "2f02ebe29573d45813e671eb304f2a00")+ set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "930495f3aeb16b25dc18a2bde45924f4")set(KLEIDIAI_FETCH_ARGSURL ${KLEIDIAI_DOWNLOAD_URL}diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cppindex d4551298f8..89e95968cd 100644--- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp+++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp@@ -3,6 +3,8 @@//// KleidiAI micro-kernels+#include <cstddef>+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h"#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h"#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h"@@ -33,6 +35,10 @@#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.h"#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.h"+#include "kai/ukernels/matmul/kai_matmul.h"+#include "kai/ukernels/matmul/kai_matmul_pack_lhs.h"+#include "kai/ukernels/matmul/kai_matmul_pack_rhs.h"+#include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h"#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h"#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h"@@ -42,8 +48,6 @@#include "kai/kai_common.h"-#include "simd-mappings.h"-#define GGML_COMMON_DECL_CPP#include "ggml-common.h"@@ -154,16 +158,6 @@ static inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/return Fn(n, k);}-template<size_t(*Fn)(size_t,size_t,size_t,size_t)>-static inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) {- return Fn(k, nr, kr, bl);-}--template<size_t(*Fn)(size_t)>-static inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) {- return Fn(k);-}-template<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const uint8_t*,const float*,void*,size_t,const struct kai_rhs_pack_qs4cxs1s0_param*)>static inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/,@@ -194,137 +188,205 @@ static inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t nFn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params);}-static const size_t INT4_PER_BYTE = 2;-static const size_t INT4_BITS = 4;-static const int Q4_0_ZERO_POINT = 8;-const size_t INT4_PER_UINT16 = 4;--static void dequantize_row_qsi4c32pscalef16(- const void *packed_data,- int32_t row_idx,- int64_t nc,- float *out,- size_t nr_pack,- size_t packed_row_stride,- size_t kr,- size_t bl,- size_t num_bytes_multiplier-) {- size_t group_idx = row_idx / nr_pack;- size_t row_in_group = row_idx % nr_pack;- const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;- size_t num_blocks = nc / bl;- const uint8_t *block_ptr = packed_group;-- for (size_t b = 0; b < num_blocks; ++b) {- uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier));- float scale = GGML_CPU_FP16_TO_FP32(scale_f16);-- const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier;- size_t num_segments = bl / kr;- size_t num_bytes_per_segment = kr / INT4_PER_BYTE;-- for (size_t s = 0; s < num_segments; ++s) {- const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment;- const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment;- for (size_t k = 0; k < num_bytes_per_segment; ++k) {- uint8_t byte = qbytes[k] ^ 0x88;- int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT;- int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT;- out[b * bl + s * num_bytes_per_segment + k] = x0 * scale;- out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale;- }- }- block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment;- }+static constexpr size_t KLEIDIAI_Q4_K_BLOCK_LENGTH = QK_K / 8;++static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks");+static_assert(sizeof(block_q4_K) == 144, "Unexpected ggml Q4_K block layout");+static_assert(offsetof(block_q4_K, scales) == 4, "Unexpected ggml Q4_K metadata offset");+static_assert(offsetof(block_q4_K, qs) == 16, "Unexpected ggml Q4_K data offset");++static struct kai_matmul_uker_config q4_k_matmul_config() {+ struct kai_matmul_uker_config config = {};+ config.format.bl = KLEIDIAI_Q4_K_BLOCK_LENGTH;+ return config;}-static void dequantize_row_qsi4c32ps1s0scalef16(- const void *packed_data,- int32_t row_idx,- int64_t k,- float *out,- size_t nr,- size_t packed_row_stride,- size_t kr,- size_t bl,- size_t num_bytes_multiplier-) {- const size_t num_blocks = k / bl;- const size_t bl4 = bl / INT4_PER_UINT16;-- size_t group_idx = row_idx / nr;- size_t row_in_group = row_idx % nr;-- const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;- const uint16_t *qdata = (const uint16_t *)packed_group;- const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier));-- for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) {- uint16_t scale_f16 = scales[row_in_group + block_idx * nr];- float scale = GGML_CPU_FP16_TO_FP32(scale_f16);-- for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) {- uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group];-- for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) {- int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT;- out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale;- }- }- }- GGML_UNUSED(kr);+template<struct kai_matmul_uker_api (*GetApi)(void)>+static size_t q4_k_get_m_step() {+ const struct kai_matmul_uker_api api = GetApi();+ const struct kai_matmul_uker_config config = q4_k_matmul_config();+ return api.get_step(&config).m;}-static void dequantize_row_qsi8cxp(- const void *packed_data,- int32_t row_idx,- int64_t k,- float *out,- size_t nr,- size_t packed_row_stride,- size_t kr,- size_t bl,- size_t num_bytes_multiplier-) {- GGML_UNUSED(bl);- GGML_UNUSED(num_bytes_multiplier);-- const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0;- const size_t group_idx = row_idx / nr;- const size_t row_in_group = row_idx % nr;-- const uint8_t * group_ptr = static_cast<const uint8_t *>(packed_data) + group_idx * packed_row_stride;- const int8_t * data_base = reinterpret_cast<const int8_t *>(group_ptr);-- const size_t num_blocks = k_internal / kr;-- for (size_t block = 0; block < num_blocks; ++block) {- const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr;- for (size_t i = 0; i < kr; ++i) {- const size_t k_idx = block * kr + i;- if (k_idx < (size_t) k) {- out[k_idx] = static_cast<float>(block_ptr[i]);- }- }- }+template<struct kai_matmul_uker_api (*GetApi)(void)>+static size_t q4_k_get_n_step() {+ const struct kai_matmul_uker_api api = GetApi();+ const struct kai_matmul_uker_config config = q4_k_matmul_config();+ return api.get_step(&config).n;+}- const uint8_t * sums_ptr = group_ptr + nr * k_internal;- GGML_UNUSED(sums_ptr);+template<size_t Value>+static size_t q4_k_get_constant() {+ return Value;+}- const float * scale_ptr = reinterpret_cast<const float *>(sums_ptr + nr * sizeof(int32_t));- const float scale = scale_ptr[row_in_group];+template<struct kai_matmul_uker_api (*GetApi)(void)>+static size_t q4_k_get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) {+ const struct kai_matmul_uker_api api = GetApi();+ const struct kai_matmul_uker_config config = q4_k_matmul_config();+ const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx };+ const struct kai_matmul_uker_dst_stride_args stride = { dst_stride };+ return api.get_dst_offset(&config, &index, &stride);+}- if (scale == 0.0f) {- for (size_t i = 0; i < (size_t) k; ++i) {- out[i] = 0.0f;- }- return;- }+template<struct kai_matmul_uker_api (*GetApi)(void)>+static size_t q4_k_get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) {+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);++ const struct kai_matmul_uker_api api = GetApi();+ const struct kai_matmul_uker_config config = q4_k_matmul_config();+ const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k };+ const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 };+ const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape);+ return api.get_lhs_offset(&config, &index, &stride);+}++template<struct kai_matmul_uker_api (*GetApi)(void)>+static size_t q4_k_get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) {+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);++ const struct kai_matmul_uker_api api = GetApi();+ const struct kai_matmul_uker_config config = q4_k_matmul_config();+ const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k };+ const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 };+ const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape);+ return api.get_rhs_offset(&config, &index, &stride);+}++template<struct kai_matmul_uker_api (*GetApi)(void)>+static void q4_k_run_matmul(+ size_t m, size_t n, size_t k, size_t bl,+ const void * lhs_packed, const void * rhs_packed,+ void * dst, size_t dst_stride_row, size_t dst_stride_col,+ float clamp_min, float clamp_max) {+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);+ GGML_ASSERT(dst_stride_col == sizeof(float));++ const struct kai_matmul_uker_api api = GetApi();+ const struct kai_matmul_uker_config config = q4_k_matmul_config();+ const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k };+ const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k };++ struct kai_matmul_uker_args args = {};+ args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP;+ args.shape = { m, n, k };+ args.operand.lhs.ptr = lhs_packed;+ args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape);+ args.operand.rhs.ptr = rhs_packed;+ args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);+ args.operand.dst.ptr = dst;+ args.operand.dst.stride.m = dst_stride_row;+ args.activation.clamp.min_ptr = &clamp_min;+ args.activation.clamp.max_ptr = &clamp_max;+ api.run(&config, &args);+}++static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() {+ struct kai_matmul_pack_lhs_uker_config config = {};+ return config;+}++static size_t q4_k_lhs_get_offset(size_t m_idx, size_t lhs_stride) {+ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();+ const struct kai_matmul_pack_lhs_uker_lhs_dim_args index = { m_idx, 0 };+ const struct kai_matmul_pack_lhs_uker_lhs_stride_args stride = { lhs_stride };+ return api.get_lhs_offset(&config, &index, &stride);+}++static size_t q4_k_lhs_get_packed_offset(+ size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);+ GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);++ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m_idx, k };+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args index = { m_idx, 0 };+ const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape);+ return api.get_lhs_packed_offset(&config, &index, &stride);+}++static size_t q4_k_lhs_get_packed_size(+ size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);+ GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);++ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m, k };+ const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape);+ return api.get_lhs_packed_size(&config, &shape, &stride);+}++static void q4_k_lhs_pack(+ size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,+ size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) {+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);+ GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);+ GGML_ASSERT(m_idx_start == 0);++ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args packed_shape = { m, k };++ struct kai_matmul_pack_lhs_uker_args args = {};+ args.flags = 0;+ args.shape = { m, k };+ args.operand.lhs.ptr = lhs;+ args.operand.lhs.stride.m = lhs_stride;+ args.operand.lhs_packed.ptr = lhs_packed;+ args.operand.lhs_packed.stride = api.get_lhs_packed_stride(&config, &packed_shape);+ api.run(&config, &args);+}- for (size_t i = 0; i < (size_t) k; ++i) {- out[i] *= scale;+static struct kai_matmul_pack_rhs_uker_config q4_k_rhs_pack_config() {+ struct kai_matmul_pack_rhs_uker_config config = {};+ return config;+}++static size_t q4_k_rhs_get_packed_size(size_t n, size_t k, size_t nr, size_t /*kr*/, size_t bl) {+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);++ const struct kai_matmul_pack_rhs_uker_api api =+ kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme();+ const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config();+ GGML_ASSERT(api.get_step(&config).n == nr);+ const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args shape = { n, k };+ const struct kai_matmul_pack_rhs_uker_rhs_packed_stride_args stride = api.get_rhs_packed_stride(&config, &shape);+ return api.get_rhs_packed_size(&config, &shape, &stride);+}++static void q4_k_rhs_pack(+ size_t num_groups, size_t n, size_t k, size_t nr, size_t /*kr*/, size_t /*sr*/, size_t bl,+ size_t rhs_stride, const void * rhs, const void * bias, const void * scale,+ void * rhs_packed, size_t extra_bytes, const void * params) {+ GGML_ASSERT(num_groups == 1);+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);+ GGML_ASSERT(bias == nullptr);+ GGML_ASSERT(scale == nullptr);+ GGML_ASSERT(extra_bytes == 0);+ GGML_ASSERT(params == nullptr);++ const struct kai_matmul_pack_rhs_uker_api api =+ kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme();+ const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config();+ GGML_ASSERT(api.get_step(&config).n == nr);+ const struct kai_matmul_pack_rhs_uker_rhs_dim_args rhs_shape = { n, k };+ const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args packed_shape = { n, k };++ struct kai_matmul_pack_rhs_uker_args args = {};+ args.flags = 0;+ args.shape = { n, k };+ args.operand.rhs.ptr = rhs;+ args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);+ if (rhs_stride != 0) {+ GGML_ASSERT(rhs_stride >= args.operand.rhs.stride.n);+ args.operand.rhs.stride.n = rhs_stride;}+ args.operand.rhs_packed.ptr = rhs_packed;+ args.operand.rhs_packed.stride = api.get_rhs_packed_stride(&config, &packed_shape);+ api.run(&config, &args);}static ggml_kleidiai_kernels gemm_gemv_kernels[] = {@@ -338,7 +400,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,@@ -359,7 +420,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,@@ -371,10 +431,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32_neon>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon,- /* .to_float = */ dequantize_row_qsi4c32ps1s0scalef16,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,},/* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16,@@ -392,7 +449,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,@@ -412,7 +468,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_lhs_offset_ex = */ nullptr,/* .get_rhs_packed_offset_ex = */ nullptr,/* .run_kernel_ex = */ nullptr,@@ -424,10 +479,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_bf16p2vlx2_f32_sme>,},/* .rhs_info = */ {- /* .packed_stride = */ nullptr,- /* .to_float = */ nullptr,/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,- /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,},/* .required_cpu = */ CPU_FEATURE_SME2,@@ -446,7 +498,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,@@ -466,7 +517,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,@@ -478,10 +528,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,- /* .to_float = */ dequantize_row_qsi4c32pscalef16,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,},/* .required_cpu = */ CPU_FEATURE_DOTPROD,@@ -499,7 +546,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,@@ -519,7 +565,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,@@ -531,10 +576,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,- /* .to_float = */ dequantize_row_qsi4c32pscalef16,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,},/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,@@ -553,7 +595,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,@@ -573,7 +614,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,@@ -585,10 +625,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,- /* .to_float = */ dequantize_row_qsi4c32pscalef16,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,},/* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,@@ -606,7 +643,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,@@ -626,7 +662,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,@@ -638,10 +673,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,- /* .to_float = */ dequantize_row_qsi4c32pscalef16,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,},/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,@@ -659,7 +691,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,@@ -679,7 +710,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,@@ -691,10 +721,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,- /* .to_float = */ dequantize_row_qsi4c32pscalef16,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,},/* .required_cpu = */ CPU_FEATURE_DOTPROD,@@ -706,6 +733,59 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {{ /* Sentinel */ }};+static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = {+ {+ /* SME2 GEMM */+ {+ /* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ /* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ /* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ /* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ /* .get_kr = */ &q4_k_get_constant<4>,+ /* .get_sr = */ &q4_k_get_constant<2>,+ /* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ /* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,+ },+ {+ /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon,+ /* .get_packed_offset_ex = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_pack_f16pmrx2_f32_neon>,+ /* .packed_size_ex = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_pack_f16pmrx2_f32_neon>,+ /* .pack_func_ex = */ &lhs_pack_void_fn10<kai_run_lhs_pack_f16pmrx2_f32_neon>,+ },+ /* SME2 GEMV */+ {+ /* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ /* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ /* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ /* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ /* .get_kr = */ &q4_k_get_constant<4>,+ /* .get_sr = */ &q4_k_get_constant<2>,+ /* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ /* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,+ },+ {+ /* .get_offset = */ q4_k_lhs_get_offset,+ /* .get_packed_offset_ex = */ q4_k_lhs_get_packed_offset,+ /* .packed_size_ex = */ q4_k_lhs_get_packed_size,+ /* .pack_func_ex = */ q4_k_lhs_pack,+ },+ {+ /* .packed_size_ex = */ q4_k_rhs_get_packed_size,+ /* .pack_func_ex = */ q4_k_rhs_pack,+ /* .repack_mode = */ RHS_REPACK_SINGLE_ONLY,+ },+ /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16,+ /* .lhs_type = */ GGML_TYPE_F32,+ /* .rhs_type = */ GGML_TYPE_Q4_K,+ /* .op_type = */ GGML_TYPE_F32,+ },+ { /* Sentinel */ }+};+static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {{/* SME2 GEMM */@@ -717,7 +797,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,@@ -737,7 +816,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,@@ -749,10 +827,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,- /* .to_float = */ dequantize_row_qsi8cxp,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,},/* .required_cpu = */ CPU_FEATURE_SME2,@@ -770,7 +845,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,@@ -790,7 +864,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,@@ -802,10 +875,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,- /* .to_float = */ dequantize_row_qsi8cxp,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,},/* .required_cpu = */ CPU_FEATURE_SME,@@ -823,7 +893,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,@@ -843,7 +912,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,@@ -855,10 +923,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,- /* .to_float = */ dequantize_row_qsi8cxp,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,},/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,@@ -876,7 +941,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,@@ -896,7 +960,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,@@ -908,10 +971,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,},/* .rhs_info = */ {- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,- /* .to_float = */ dequantize_row_qsi8cxp,/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,},/* .required_cpu = */ CPU_FEATURE_DOTPROD,@@ -933,7 +993,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,@@ -953,7 +1012,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,/* .run_kernel_ex = */ &kernel_run_lhs_stride_fn10<kai_run_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,@@ -965,10 +1023,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .pack_func_ex = */ nullptr,},/* .rhs_info = */ {- /* .packed_stride = */ nullptr,- /* .to_float = */ nullptr,/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,- /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,},/* .required_cpu = */ CPU_FEATURE_SME2,@@ -986,7 +1041,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,@@ -1006,7 +1060,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,/* .get_lhs_offset_ex = */ nullptr,/* .get_rhs_packed_offset_ex = */ nullptr,/* .run_kernel_ex = */ nullptr,@@ -1018,10 +1071,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,},/* .rhs_info = */ {- /* .packed_stride = */ nullptr,- /* .to_float = */ nullptr,/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,- /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,},/* .required_cpu = */ CPU_FEATURE_SME,@@ -1049,8 +1099,23 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, creturn false;};- if (tensor->src[0]->type == GGML_TYPE_Q8_0) {- try_table(gemm_gemv_kernels_q8);+ if (tensor->src[0]->type == GGML_TYPE_Q4_K) {+ try_table(gemm_gemv_kernels_q4_k);+ } else if (tensor->src[0]->type == GGML_TYPE_Q8_0 || tensor->src[0]->type == GGML_TYPE_Q6_K) {+ const ggml_type original_type = tensor->src[0]->type;+ if (original_type == GGML_TYPE_Q6_K) {+ for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) {+ if ((cpu_features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu &&+ gemm_gemv_kernels_q8[i].lhs_type == tensor->src[1]->type &&+ gemm_gemv_kernels_q8[i].rhs_type == GGML_TYPE_Q8_0 &&+ gemm_gemv_kernels_q8[i].op_type == tensor->type) {+ kernel = &gemm_gemv_kernels_q8[i];+ break;+ }+ }+ } else {+ try_table(gemm_gemv_kernels_q8);+ }} else if (tensor->src[0]->type == GGML_TYPE_F32) {try_table(ggml_kleidiai_kernels_f32);} else {@@ -1075,6 +1140,19 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features)return kernels;}+ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features) {+ ggml_kleidiai_kernels * kernels = nullptr;++ for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q4_k) - 1; ++i) {+ if ((features & gemm_gemv_kernels_q4_k[i].required_cpu) == gemm_gemv_kernels_q4_k[i].required_cpu) {+ kernels = &gemm_gemv_kernels_q4_k[i];+ break;+ }+ }++ return kernels;+}+ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) {ggml_kleidiai_kernels * kernels = nullptr;diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.hindex 1da8610eae..308ea94518 100644--- a/ggml/src/ggml-cpu/kleidiai/kernels.h+++ b/ggml/src/ggml-cpu/kleidiai/kernels.h@@ -33,7 +33,6 @@ struct kernel_info {size_t (*get_sr)(void);size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride);- size_t (*get_dst_size)(size_t m, size_t n);size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl);@@ -59,21 +58,12 @@ struct lhs_packing_info {enum rhs_repack_mode {RHS_REPACK_PER_KERNEL,- RHS_REPACK_SHARED,RHS_REPACK_SINGLE_ONLY,};struct rhs_packing_info {- size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl);-- void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out,- size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl,- size_t num_bytes_multiplier);-size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl);- size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl);-void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params);@@ -97,5 +87,6 @@ struct ggml_kleidiai_kernels {ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor);ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features);+ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features);ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features);ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features);diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cppindex 1384ca9278..dd9bcbf634 100644--- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp+++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp@@ -43,6 +43,7 @@#include "ggml-impl.h"#include "ggml-feats.h"#include "ggml-backend-impl.h"+#include "ggml-quants.h"#include "ggml-threading.h"#include "traits.h"@@ -57,16 +58,20 @@ static constexpr int GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2;static constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC = 0x4b4c4149; // "KLAI"static constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION = 1;static constexpr size_t GGML_KLEIDIAI_PACK_ALIGN = 64;+static constexpr size_t GGML_KLEIDIAI_Q4_K_BL = QK_K / 8;++static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks");struct ggml_kleidiai_context {cpu_feature features;ggml_kleidiai_kernels * kernels_q4;+ ggml_kleidiai_kernels * kernels_q4k;ggml_kleidiai_kernels * kernels_q8;ggml_kleidiai_kernels * kernels_f32;int sme_thread_cap; // <= 0 means "SME disabled/unknown"int thread_hint; // <= 0 means "no hint"int chunk_multiplier;-} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 };+} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, nullptr, 0, -1, 4 };static inline bool is_sme_family(cpu_feature f) {return (f & (CPU_FEATURE_SME | CPU_FEATURE_SME2)) != CPU_FEATURE_NONE;@@ -387,6 +392,7 @@ static void init_kleidiai_context(void) {// Kernel selectionctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features);+ ctx.kernels_q4k = ggml_kleidiai_select_kernels_q4_k(ctx.features);ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features);ctx.kernels_f32 = ggml_kleidiai_select_kernels_f32(ctx.features);@@ -402,6 +408,12 @@ static void init_kleidiai_context(void) {GGML_LOG_INFO("kleidiai: primary q8 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q8->required_cpu));}+ if (!ctx.kernels_q4k) {+ GGML_LOG_INFO("kleidiai: no compatible q4_k kernels found for CPU features mask %d\n", (int) ctx.features);+ } else {+ GGML_LOG_INFO("kleidiai: primary q4_k kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q4k->required_cpu));+ }+if (!ctx.kernels_f32) {GGML_LOG_INFO("kleidiai: no compatible f32 kernels found for CPU features mask %d\n", (int)ctx.features);} else {@@ -410,6 +422,7 @@ static void init_kleidiai_context(void) {const bool has_selected_sme_family_kernel =(ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) ||+ (ctx.kernels_q4k && is_sme_family(ctx.kernels_q4k->required_cpu)) ||(ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) ||(ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu));ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0;@@ -486,6 +499,8 @@ static inline kleidiai_block_args kleidiai_get_block_args(ggml_type rhs_type) {switch (rhs_type) {case GGML_TYPE_Q4_0:return { QK4_0, QK4_0, QK4_0 };+ case GGML_TYPE_Q4_K:+ return { GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL };case GGML_TYPE_Q8_0:return { 0, 0, QK8_0 };default:@@ -556,6 +571,10 @@ static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() {return ctx.kernels_q4;}+static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4k() {+ return ctx.kernels_q4k;+}+static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() {return ctx.kernels_q8;}@@ -610,6 +629,12 @@ static int kleidiai_collect_q4_chain(std::array<ggml_kleidiai_kernels *, GGML_KL[&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_0(mask); });}+static int kleidiai_collect_q4k_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {+ ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4k();+ return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,+ [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_k(mask); });+}+static int kleidiai_collect_q8_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8();return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,@@ -662,8 +687,10 @@ class tensor_traits : public ggml::cpu::tensor_traits {const size_t n = op->src[0]->ne[1];const size_t m = op->src[1]->ne[1];- if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {- const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0;+ if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K ||+ op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K) {+ const kleidiai_block_args block_args = kleidiai_get_block_args(kernel_chain[0]->rhs_type);+ const size_t qk = block_args.pack_bl;size_t cursor = 0;bool any_slot = false;@@ -789,17 +816,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override {if (dst->op == GGML_OP_MUL_MAT) {- if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {+ if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K ||+ dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K) {return compute_forward_qx(params, dst);} else if (dst->src[0]->type == GGML_TYPE_F32) {return compute_forward_f32(params, dst);} else if (dst->src[0]->type == GGML_TYPE_F16) {return compute_forward_fp16(params, dst);}- } else if (dst->op == GGML_OP_GET_ROWS) {- if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {- return compute_forward_get_rows(params, dst);- }}return false;}@@ -870,9 +894,12 @@ class tensor_traits : public ggml::cpu::tensor_traits {const bool disable_chunking = ggml_is_numa();GGML_ASSERT(n <= (size_t) INT_MAX);- for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) {- const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];- uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];+ const int64_t batch_size = ne12 * ne13;+ for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) {+ const int64_t i12 = batch_idx % ne12;+ const int64_t i13 = batch_idx / ne12;+ const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + i12 * src1->nb[2] + i13 * src1->nb[3];+ uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + i12 * dst->nb[2] + i13 * dst->nb[3];if (!use_gemv) {const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr);@@ -947,7 +974,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {current_col = ggml_threadpool_chunk_add(params->threadpool, (int) chunk_cols);}- if (batch_idx != ne12 - 1) {+ if (batch_idx != batch_size - 1) {ggml_barrier(params->threadpool);}}@@ -1107,7 +1134,8 @@ class tensor_traits : public ggml::cpu::tensor_traits {}bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) {- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);+ GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K ||+ dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K);const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];@@ -1265,7 +1293,9 @@ class tensor_traits : public ggml::cpu::tensor_traits {}}- int sme_cap = kleidiai_sme_thread_cap();+ const bool sme_single_only = sme_slot != -1 &&+ runtime[sme_slot].kernels->rhs_info.repack_mode == RHS_REPACK_SINGLE_ONLY;+ int sme_cap = sme_single_only ? nth_total : kleidiai_sme_thread_cap();if (sme_cap < 0) {sme_cap = nth_total;}@@ -1301,23 +1331,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {threads_remaining -= share;}- if (threads_remaining > 0) {- const int fallback_slot = (sme_slot != -1) ? sme_slot : 0;- runtime[fallback_slot].assigned_threads += threads_remaining;- threads_remaining = 0;- }-int thread_cursor = 0;for (int i = 0; i < runtime_count; ++i) {runtime[i].thread_begin = thread_cursor;thread_cursor += runtime[i].assigned_threads;runtime[i].thread_end = thread_cursor;}-- if (thread_cursor < nth_total && runtime_count > 0) {- runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor;- runtime[runtime_count - 1].thread_end = nth_total;- }+ GGML_ASSERT(thread_cursor > 0);+ const int active_threads = thread_cursor;int local_slot = -1;int local_ith = 0;@@ -1328,10 +1349,6 @@ class tensor_traits : public ggml::cpu::tensor_traits {break;}}- if (local_slot == -1) {- return false;- }-const size_t k = ne00;const size_t m = ne11;const size_t n = ne01;@@ -1363,7 +1380,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {const bool disable_chunking = ggml_is_numa();const size_t chunk_multiplier = std::max(1, ctx.chunk_multiplier);- const size_t chunk_divisor = (nth_total == 1 || disable_chunking) ? (size_t)nth_total : (size_t)nth_total * chunk_multiplier;+ const size_t chunk_divisor = (active_threads == 1 || disable_chunking) ? (size_t) active_threads : (size_t) active_threads * chunk_multiplier;size_t chunk_cols = align_up(std::max<size_t>(1, ceil_div_size(n, chunk_divisor)), common_step);if (chunk_cols == 0) {chunk_cols = common_step;@@ -1397,7 +1414,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];- if (runtime[local_slot].assigned_threads > 0) {+ if (local_slot != -1) {runtime_slot & slot = runtime[local_slot];const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr);int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads;@@ -1438,28 +1455,30 @@ class tensor_traits : public ggml::cpu::tensor_traits {}if (ith_total == 0) {- ggml_threadpool_chunk_set(params->threadpool, nth_total);+ ggml_threadpool_chunk_set(params->threadpool, active_threads);}// Publishes both LHS packing and the initialized dynamic chunk queue.ggml_barrier(params->threadpool);- runtime_slot & slot = runtime[local_slot];- int current_chunk = ith_total;- while (current_chunk < nchunk) {- const size_t global_start = (size_t)current_chunk * chunk_cols;- if (global_start >= n) {- break;- }+ if (local_slot != -1) {+ runtime_slot & slot = runtime[local_slot];+ int current_chunk = ith_total;+ while (current_chunk < nchunk) {+ const size_t global_start = (size_t)current_chunk * chunk_cols;+ if (global_start >= n) {+ break;+ }- const size_t cols = std::min(chunk_cols, n - global_start);- if (cols > 0) {- // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths;- // only non-tail chunks are guaranteed to be n_step-aligned.- run_chunk(slot, global_start, cols, dst_batch_base);- }+ const size_t cols = std::min(chunk_cols, n - global_start);+ if (cols > 0) {+ // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths;+ // only non-tail chunks are guaranteed to be n_step-aligned.+ run_chunk(slot, global_start, cols, dst_batch_base);+ }- current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);+ current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);+ }}if (batch_idx != ne12 - 1) {@@ -1470,101 +1489,11 @@ class tensor_traits : public ggml::cpu::tensor_traits {return true;}- bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) {- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);- const ggml_tensor * src0 = dst->src[0];- const ggml_tensor * src1 = dst->src[1];-- GGML_TENSOR_BINARY_OP_LOCALS-- const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);- const bool has_header = kleidiai_is_weight_header_valid(header);-- std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;- const bool want_q8 = src0->type == GGML_TYPE_Q8_0;- const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)- : kleidiai_collect_q4_chain(kernel_chain);-- ggml_kleidiai_kernels * kernels = nullptr;- const uint8_t * packed_base = static_cast<const uint8_t *>(src0->data);-- if (has_header && chain_count > 0) {- int select_slot = 0;- if (select_slot >= header->slot_count) {- select_slot = header->slot_count - 1;- }- if (select_slot >= 0 && select_slot < chain_count) {- kernels = kernel_chain[select_slot];- const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot);- if (slot_ptr) {- packed_base = slot_ptr;- }- }- }-- if (!kernels && chain_count > 0) {- kernels = kernel_chain[0];- if (has_header) {- const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0);- if (slot_ptr) {- packed_base = slot_ptr;- }- }- }-- if (!kernels) {- return false;- }-- rhs_packing_info * rhs_info = &kernels->rhs_info;- kernel_info * kernel = &kernels->gemm;- if (!rhs_info->to_float || !kernel->get_nr) {- return false;- }-- const int64_t nc = ne00;- const int64_t nr = ggml_nelements(src1);-- const ggml_type rhs_type = kernels->rhs_type;- size_t block_len = 0;- size_t num_bytes_multiplier = 0;- if (rhs_type == GGML_TYPE_Q4_0) {- block_len = QK4_0;- num_bytes_multiplier = sizeof(uint16_t);- } else if (rhs_type == GGML_TYPE_Q8_0) {- block_len = QK8_0;- num_bytes_multiplier = sizeof(float);- } else {- return false;- }-- const size_t block_rows = kernel->get_nr();- const size_t kr = kernel->get_kr();-- const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len);-- const int ith = params->ith;- const int nth = params->nth;-- const int dr = (nr + nth - 1) / nth;- const int ir0 = dr * ith;- const int ir1 = MIN(ir0 + dr, nr);-- for (int64_t i = ir0; i < ir1; ++i) {- GGML_ASSERT(src1->type == GGML_TYPE_I32);- int64_t row_idx = ((const int32_t *)src1->data)[i];- GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]);-- float *out = (float *)((char *)dst->data + i * nb1);- rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier);- }-- return true;- }-public:int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) {- GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_F32);+ GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q4_K ||+ tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K ||+ tensor->type == GGML_TYPE_F32);const size_t n = tensor->ne[1];const size_t k = tensor->ne[0];@@ -1582,16 +1511,19 @@ public:cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;- const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;+ const bool want_q4k = tensor->type == GGML_TYPE_Q4_K;+ const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K;const bool want_f32 = tensor->type == GGML_TYPE_F32;const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain): want_q8 ? kleidiai_collect_q8_chain(kernel_chain)- : kleidiai_collect_q4_chain(kernel_chain);+ : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain)+ : kleidiai_collect_q4_chain(kernel_chain);const bool allow_fallback = kleidiai_pack_fallback_allowed();std::vector<int8_t> qdata;std::vector<float> scales;std::vector<float> bias;+ std::vector<float> row_f32;if (want_q8 && slot_total > 0) {qdata.resize(n * k, 0);@@ -1600,21 +1532,38 @@ public:const size_t row_stride = tensor->nb[1];const size_t k_blocks = (k + QK8_0 - 1) / QK8_0;+ if (tensor->type == GGML_TYPE_Q6_K) {+ if (k % QK_K != 0 || k % QK8_0 != 0) {+ return -1;+ }+ row_f32.resize(k);+ }+for (size_t row = 0; row < n; ++row) {- const auto * row_blocks = reinterpret_cast<const block_q8_0 *>(- static_cast<const uint8_t *>(data) + row * row_stride);+ const uint8_t * row_data = static_cast<const uint8_t *>(data) + row * row_stride;++ if (tensor->type == GGML_TYPE_Q6_K) {+ dequantize_row_q6_K(reinterpret_cast<const block_q6_K *>(row_data), row_f32.data(), k);+ }float max_abs = 0.0f;- for (size_t block = 0; block < k_blocks; ++block) {- const block_q8_0 & blk = row_blocks[block];- const float d = GGML_FP16_TO_FP32(blk.d);- for (size_t l = 0; l < QK8_0; ++l) {- const size_t linear_idx = block * QK8_0 + l;- if (linear_idx >= k) {- break;+ if (tensor->type == GGML_TYPE_Q6_K) {+ for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) {+ max_abs = std::max(max_abs, std::fabs(row_f32[linear_idx]));+ }+ } else {+ const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data);+ for (size_t block = 0; block < k_blocks; ++block) {+ const block_q8_0 & blk = row_blocks[block];+ const float d = GGML_FP16_TO_FP32(blk.d);+ for (size_t l = 0; l < QK8_0; ++l) {+ const size_t linear_idx = block * QK8_0 + l;+ if (linear_idx >= k) {+ break;+ }+ const float value = d * static_cast<float>(blk.qs[l]);+ max_abs = std::max(max_abs, std::fabs(value));}- const float value = d * static_cast<float>(blk.qs[l]);- max_abs = std::max(max_abs, std::fabs(value));}}@@ -1622,19 +1571,28 @@ public:scales[row] = scale;const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f;- for (size_t block = 0; block < k_blocks; ++block) {- const block_q8_0 & blk = row_blocks[block];- const float d = GGML_FP16_TO_FP32(blk.d);- for (size_t l = 0; l < QK8_0; ++l) {- const size_t linear_idx = block * QK8_0 + l;- if (linear_idx >= k) {- break;- }- const float value = d * static_cast<float>(blk.qs[l]);- int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0;+ if (tensor->type == GGML_TYPE_Q6_K) {+ for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) {+ int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(row_f32[linear_idx] * inv_scale)) : 0;q = std::clamp(q, -127, 127);qdata[row * k + linear_idx] = static_cast<int8_t>(q);}+ } else {+ const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data);+ for (size_t block = 0; block < k_blocks; ++block) {+ const block_q8_0 & blk = row_blocks[block];+ const float d = GGML_FP16_TO_FP32(blk.d);+ for (size_t l = 0; l < QK8_0; ++l) {+ const size_t linear_idx = block * QK8_0 + l;+ if (linear_idx >= k) {+ break;+ }+ const float value = d * static_cast<float>(blk.qs[l]);+ int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0;+ q = std::clamp(q, -127, 127);+ qdata[row * k + linear_idx] = static_cast<int8_t>(q);+ }+ }}}}@@ -1660,6 +1618,7 @@ public:const ggml_type rhs_type = kernels->rhs_type;const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :+ rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX;if (block_len == SIZE_MAX) {continue;@@ -1677,6 +1636,9 @@ public:rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0,static_cast<const uint8_t *>(data), nullptr, nullptr,dst_ptr, 0, ¶ms);+ } else if (rhs_type == GGML_TYPE_Q4_K) {+ rhs_info->pack_func_ex(1, n, k, nr, kr, sr, GGML_KLEIDIAI_Q4_K_BL, tensor->nb[1],+ data, nullptr, nullptr, dst_ptr, 0, nullptr);} else if (rhs_type == GGML_TYPE_Q8_0) {struct kai_rhs_pack_qsi8cx_params params;params.lhs_zero_point = 1;@@ -1719,7 +1681,6 @@ static enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backentensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor);return GGML_STATUS_SUCCESS;- GGML_UNUSED(buffer);}static void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,@@ -1762,7 +1723,9 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_bstatic size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {GGML_UNUSED(buft);- if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_F32) {+ if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q4_K &&+ tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_Q6_K &&+ tensor->type != GGML_TYPE_F32) {return ggml_nbytes(tensor);}@@ -1773,11 +1736,13 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;- const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;+ const bool want_q4k = tensor->type == GGML_TYPE_Q4_K;+ const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K;const bool want_f32 = tensor->type == GGML_TYPE_F32;const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain): want_q8 ? kleidiai_collect_q8_chain(kernel_chain)- : kleidiai_collect_q4_chain(kernel_chain);+ : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain)+ : kleidiai_collect_q4_chain(kernel_chain);const bool allow_fallback = kleidiai_pack_fallback_allowed();size_t slot_count = 0;@@ -1797,6 +1762,7 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_const ggml_type rhs_type = kernels->rhs_type;const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :+ rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX;if (block_len == SIZE_MAX) {@@ -1818,6 +1784,11 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_namespace ggml::cpu::kleidiai {class extra_buffer_type : ggml::cpu::extra_buffer_type {bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {+ // GET_ROWS needs the original weights in a native buffer.+ if (op->op != GGML_OP_MUL_MAT) {+ return false;+ }+std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain);const bool src0_is_kleidiai =@@ -1826,8 +1797,9 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {op->src[0]->buffer->buft->context == this &&slot_total > 0;- if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) &&- (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_F32) &&+ if ((op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K ||+ op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K ||+ op->src[0]->type == GGML_TYPE_F32) &&src0_is_kleidiai) {if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) {return false;@@ -1835,6 +1807,12 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) {return false;}+ if (op->src[0]->type == GGML_TYPE_Q6_K && ctx.kernels_q8 == nullptr) {+ return false;+ }+ if (op->src[0]->type == GGML_TYPE_Q4_K && ctx.kernels_q4k == nullptr) {+ return false;+ }if (op->src[0]->type == GGML_TYPE_F32 && ctx.kernels_f32 == nullptr) {return false;}@@ -1842,8 +1820,18 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {return false;}- if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {- if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) &&+ if (op->src[0]->type == GGML_TYPE_Q4_K) {+ if (op->op != GGML_OP_MUL_MAT || op->src[1]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) {+ return false;+ }+ return ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1 &&+ op->src[1]->nb[0] == sizeof(float) &&+ (op->src[1]->ne[1] == 1 || op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float));+ }++ if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 ||+ op->src[0]->type == GGML_TYPE_Q6_K) {+ if (op->src[1]->type == GGML_TYPE_F32 &&ggml_ne(op->src[1], 3) == 1) {return true;}@@ -1861,21 +1849,22 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {}ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {- if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) {+ if (op->op == GGML_OP_MUL_MAT) {if (op->src[0]->buffer && op->src[0]->buffer->buft->context == this) {return (ggml::cpu::tensor_traits *) op->src[0]->extra;} else {- // KleidiAI only has kernels for Q4_0 and Q8_0. For a quantized weight of any+ // KleidiAI only has kernels for Q4_0, Q4_K, Q8_0 and Q6_K. For a quantized weight of any// other type (K-quants, IQ) it declines the op and returns nullptr below, so// KleidiAI does not accelerate it. Another CPU backend may still take the op,// and this can run during graph planning, so the message says what KleidiAI// did rather than what ends up executing. Warn once per process.if (ggml_is_quantized(op->src[0]->type) &&- op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q8_0) {+ op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q4_K &&+ op->src[0]->type != GGML_TYPE_Q8_0 && op->src[0]->type != GGML_TYPE_Q6_K) {static std::atomic<bool> warned(false);if (!warned.exchange(true)) {GGML_LOG_WARN("kleidiai: no kernel for tensor type %s, not accelerated by KleidiAI "- "(kernels available for Q4_0 and Q8_0)\n",+ "(kernels available for Q4_0, Q4_K, Q8_0 and Q6_K)\n",ggml_type_name(op->src[0]->type));}}