LLM inference in C/C++
C++ ggml
hexagon: Support for K-Quants Q4_K and Q6_K (#28994)
Marco Colombo1ec81880944a
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 36b9f817cb..3f14956453 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -21,6 +21,7 @@#include <queue>#include <deque>#include <algorithm>+#include <cmath>#ifdef _WIN32# define WIN32_LEAN_AND_MEAN@@ -250,7 +251,20 @@ enum ggml_hexagon_tensor_flags {static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) {return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL ||- type == GGML_TYPE_MXFP4;+ type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K ||+ type == GGML_TYPE_Q4_K;+}++// Size of one repacked row in the DSP tiled layout. The Q6_K and Q4_K tiles store uncompressed scales/mins,+// so they are larger than the ggml blocks. For the other repack types the tile has the same size as the ggml blocks.+static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne0) {+ if (type == GGML_TYPE_Q6_K) {+ return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q6_K / 32);+ }+ if (type == GGML_TYPE_Q4_K) {+ return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q4_1 / 32);+ }+ return ggml_row_size(type, ne0);}static inline bool ggml_hexagon_is_hmx_weight_type(enum ggml_type type) {@@ -1302,6 +1316,377 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset}}+// unsigned 6-bit value (0..63) of element e of a Q6_K block, same bit layout as dequantize_row_q6_K+static inline uint8_t q6_K_get_quant(const block_q6_K * b, int e) {+ const int c = e / 128;+ const int w = e % 128;+ const int g = w / 32;+ const int l = w % 32;+ const uint8_t * ql = b->ql + c * 64;+ const uint8_t * qh = b->qh + c * 32;+ uint8_t lo, hi;+ switch (g) {+ case 0: lo = ql[l] & 0xF; hi = (qh[l] >> 0) & 3; break;+ case 1: lo = ql[l + 32] & 0xF; hi = (qh[l] >> 2) & 3; break;+ case 2: lo = ql[l] >> 4; hi = (qh[l] >> 4) & 3; break;+ default: lo = ql[l + 32] >> 4; hi = (qh[l] >> 6) & 3; break;+ }+ return (uint8_t) (lo | (hi << 4));+}++// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q6_K in htp/matmul-ops.h+static void repack_q6_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset == 0);++ const block_q6_K * src_matrix = (const block_q6_K *) data;+ int64_t ne0 = t->ne[0];+ int64_t ne1 = t->ne[1];+ int64_t ne2 = t->ne[2];+ int64_t ne3 = t->ne[3];+ int64_t ne0_padded = hex_round_up(ne0, 32);+ int64_t ne1_padded = hex_round_up(ne1, 32);++ GGML_ASSERT(ne0 % QK_K == 0);++ const int n_col_tiles = ne1_padded / 32;+ const int n_k_tiles = ne0_padded / 32;+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K;+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;++ const int64_t sb_per_row = ne0 / QK_K;++ for (int i3 = 0; i3 < ne3; i3++) {+ for (int i2 = 0; i2 < ne2; i2++) {+ const block_q6_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);+ uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;++ memset(matrix_dst, 0, matrix_size); // padding rows and the OR-ed nibbles below need zeroed tiles++ for (int64_t r = 0; r < ne1; r++) {+ const int ct = (int) (r / 32);+ const int row = (int) (r % 32);+ const block_q6_K * src_row = src_slice + r * sb_per_row;++ for (int kt = 0; kt < n_k_tiles; kt++) {+ const int kt_local = kt % 8; // k-tile within the super-block+ const block_q6_K * b = &src_row[kt / 8];+ const float d = GGML_FP16_TO_FP32(b->d);++ uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;+ uint8_t * lo_pl = tile;+ uint8_t * hi_pl = tile + 512;+ ggml_half * sc_pl = (ggml_half *) (tile + 768);++ for (int lk = 0; lk < 32; lk++) {+ const uint8_t q6 = q6_K_get_quant(b, kt_local * 32 + lk);+ const int g = lk >> 2;+ const int pos = row * 4 + (lk & 3);+ lo_pl[(g >> 1) * 128 + pos] |= (uint8_t) ((q6 & 0xF) << ((g & 1) * 4));+ hi_pl[(g >> 2) * 128 + pos] |= (uint8_t) ((q6 >> 4) << ((g & 3) * 2));+ }+ for (int sub = 0; sub < 2; sub++) {+ sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) b->scales[kt_local * 2 + sub]);+ }+ }+ }+ }+ }++ GGML_UNUSED(size);+}++// Reverse of repack_q6_K_tiled. Unpacks quants losslessly and normalizes sub-block scales. Read-back only.+static void repack_tiled_q6_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {+ GGML_ASSERT(offset == 0);++ block_q6_K * dst_matrix = (block_q6_K *) data;+ int64_t ne0 = t->ne[0];+ int64_t ne1 = t->ne[1];+ int64_t ne2 = t->ne[2];+ int64_t ne3 = t->ne[3];+ int64_t ne0_padded = hex_round_up(ne0, 32);+ int64_t ne1_padded = hex_round_up(ne1, 32);++ GGML_ASSERT(ne0 % QK_K == 0);++ const int n_col_tiles = ne1_padded / 32;+ const int n_k_tiles = ne0_padded / 32;+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K;+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;++ const int64_t sb_per_row = ne0 / QK_K;++ for (int i3 = 0; i3 < ne3; i3++) {+ for (int i2 = 0; i2 < ne2; i2++) {+ block_q6_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);+ const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;++ for (int64_t r = 0; r < ne1; r++) {+ const int ct = (int) (r / 32);+ const int row = (int) (r % 32);+ block_q6_K * dst_row = dst_slice + r * sb_per_row;++ for (int64_t sb = 0; sb < sb_per_row; sb++) {+ block_q6_K * b = &dst_row[sb];+ memset(b, 0, sizeof(block_q6_K));++ float sub_scales[16];+ for (int kt_local = 0; kt_local < 8; kt_local++) {+ const int kt = sb * 8 + kt_local;+ const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;+ const uint8_t * lo_pl = tile;+ const uint8_t * hi_pl = tile + 512;+ const ggml_half * sc_pl = (const ggml_half *) (tile + 768);++ const int c = kt_local / 4;+ const int g = kt_local % 4;+ uint8_t * ql = b->ql + c * 64;+ uint8_t * qh = b->qh + c * 32;++ for (int lk = 0; lk < 32; lk++) {+ const int g_tile = lk >> 2;+ const int pos = row * 4 + (lk & 3);+ const uint8_t lo = (lo_pl[(g_tile >> 1) * 128 + pos] >> ((g_tile & 1) * 4)) & 0xF;+ const uint8_t hi = (hi_pl[(g_tile >> 2) * 128 + pos] >> ((g_tile & 3) * 2)) & 3;++ switch (g) {+ case 0:+ ql[lk] |= lo;+ qh[lk] |= (hi << 0);+ break;+ case 1:+ ql[lk + 32] |= lo;+ qh[lk] |= (hi << 2);+ break;+ case 2:+ ql[lk] |= (lo << 4);+ qh[lk] |= (hi << 4);+ break;+ default:+ ql[lk + 32] |= (lo << 4);+ qh[lk] |= (hi << 6);+ break;+ }+ }++ for (int sub = 0; sub < 2; sub++) {+ sub_scales[kt_local * 2 + sub] = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]);+ }+ }++ float max_abs_scale = 0.0f;+ for (int s = 0; s < 16; s++) {+ float abs_scale = fabsf(sub_scales[s]);+ if (abs_scale > max_abs_scale) {+ max_abs_scale = abs_scale;+ }+ }++ if (max_abs_scale == 0.0f) {+ b->d = GGML_FP32_TO_FP16(0.0f);+ memset(b->scales, 0, sizeof(b->scales));+ } else {+ float d_flt = max_abs_scale / 127.0f;+ b->d = GGML_FP32_TO_FP16(d_flt);+ float d_actual = GGML_FP16_TO_FP32(b->d);+ float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f;+ for (int s = 0; s < 16; s++) {+ int sc = (int) roundf(sub_scales[s] * inv_d);+ b->scales[s] = (int8_t) (std::max)(-128, (std::min)(127, sc));+ }+ }+ }+ }+ }+ }++ GGML_UNUSED(size);+}++static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {+ if (j < 4) {+ *d = q[j] & 63;+ *m = q[j + 4] & 63;+ } else {+ *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);+ *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);+ }+}++// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q4_1 in htp/matmul-ops.h+static void repack_q4_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset == 0);++ const block_q4_K * src_matrix = (const block_q4_K *) data;+ int64_t ne0 = t->ne[0];+ int64_t ne1 = t->ne[1];+ int64_t ne2 = t->ne[2];+ int64_t ne3 = t->ne[3];+ int64_t ne0_padded = hex_round_up(ne0, 32);+ int64_t ne1_padded = hex_round_up(ne1, 32);++ GGML_ASSERT(ne0 % QK_K == 0);++ const int n_col_tiles = ne1_padded / 32;+ const int n_k_tiles = ne0_padded / 32;+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1;+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;++ const int64_t sb_per_row = ne0 / QK_K;++ for (int i3 = 0; i3 < ne3; i3++) {+ for (int i2 = 0; i2 < ne2; i2++) {+ const block_q4_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);+ uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;++ memset(matrix_dst, 0, matrix_size);++ for (int64_t r = 0; r < ne1; r++) {+ const int ct = (int) (r / 32);+ const int row = (int) (r % 32);+ const block_q4_K * src_row = src_slice + r * sb_per_row;++ for (int kt = 0; kt < n_k_tiles; kt++) {+ const int kt_local = kt % 8;+ const block_q4_K * b = &src_row[kt / 8];+ const float d = GGML_FP16_TO_FP32(b->d);+ const float dmin = GGML_FP16_TO_FP32(b->dmin);++ uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;++ uint8_t sc, m;+ get_scale_min_k4(kt_local, b->scales, &sc, &m);++ const float D = d * (float) sc;+ const float M = -dmin * (float) m;++ const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;+ const int shift = (kt_local & 1) ? 4 : 0;++ for (int cp = 0; cp < 16; cp++) {+ const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F;+ const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F;+ tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0);+ }++ ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);+ scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(D);+ scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(M);+ }+ }+ }+ }++ GGML_UNUSED(size);+}++// Reverse of repack_q4_K_tiled. Unpacks quants and normalizes scales/mins. Read-back only.+static void repack_tiled_q4_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {+ GGML_ASSERT(offset == 0);++ block_q4_K * dst_matrix = (block_q4_K *) data;+ int64_t ne0 = t->ne[0];+ int64_t ne1 = t->ne[1];+ int64_t ne2 = t->ne[2];+ int64_t ne3 = t->ne[3];+ int64_t ne0_padded = hex_round_up(ne0, 32);+ int64_t ne1_padded = hex_round_up(ne1, 32);++ GGML_ASSERT(ne0 % QK_K == 0);++ const int n_col_tiles = ne1_padded / 32;+ const int n_k_tiles = ne0_padded / 32;+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1;+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;++ const int64_t sb_per_row = ne0 / QK_K;++ for (int i3 = 0; i3 < ne3; i3++) {+ for (int i2 = 0; i2 < ne2; i2++) {+ block_q4_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);+ const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;++ for (int64_t r = 0; r < ne1; r++) {+ const int ct = (int) (r / 32);+ const int row = (int) (r % 32);+ block_q4_K * dst_row = dst_slice + r * sb_per_row;++ for (int64_t sb = 0; sb < sb_per_row; sb++) {+ block_q4_K * b = &dst_row[sb];+ memset(b, 0, sizeof(block_q4_K));++ float sub_scales[8];+ float sub_mins[8];++ for (int kt_local = 0; kt_local < 8; kt_local++) {+ const int kt = sb * 8 + kt_local;+ const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;+ const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);++ uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;+ const int shift = (kt_local & 1) ? 4 : 0;++ for (int cp = 0; cp < 16; cp++) {+ const uint8_t val = tile_src[cp * 32 + row];+ const uint8_t q0 = val & 0x0F;+ const uint8_t q1 = val >> 4;+ qs_sub[2 * cp + 0] |= (uint8_t) (q0 << shift);+ qs_sub[2 * cp + 1] |= (uint8_t) (q1 << shift);+ }++ const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]);+ const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]);+ sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f;+ sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f;+ }++ float max_scale = 0.0f;+ float max_min = 0.0f;+ for (int j = 0; j < 8; j++) {+ if (sub_scales[j] > max_scale) max_scale = sub_scales[j];+ if (sub_mins[j] > max_min) max_min = sub_mins[j];+ }++ float inv_scale = 0.0f;+ if (max_scale > 0.0f) {+ b->d = GGML_FP32_TO_FP16(max_scale / 63.0f);+ const float d_actual = GGML_FP16_TO_FP32(b->d);+ inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f;+ } else {+ b->d = GGML_FP32_TO_FP16(0.0f);+ }++ float inv_min = 0.0f;+ if (max_min > 0.0f) {+ b->dmin = GGML_FP32_TO_FP16(max_min / 63.0f);+ const float dmin_actual = GGML_FP16_TO_FP32(b->dmin);+ inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f;+ } else {+ b->dmin = GGML_FP32_TO_FP16(0.0f);+ }++ for (int j = 0; j < 8; j++) {+ uint8_t ls = (uint8_t) roundf(inv_scale * sub_scales[j]);+ uint8_t lm = (uint8_t) roundf(inv_min * sub_mins[j]);+ ls = (std::min)((uint8_t) 63, ls);+ lm = (std::min)((uint8_t) 63, lm);+ if (j < 4) {+ b->scales[j] = ls;+ b->scales[j + 4] = lm;+ } else {+ b->scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4);+ b->scales[j - 4] |= ((ls >> 4) << 6);+ b->scales[j - 0] |= ((lm >> 4) << 6);+ }+ }+ }+ }+ }+ }++ GGML_UNUSED(size);+}+static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) {switch (tensor->type) {case GGML_TYPE_Q4_0:@@ -1312,6 +1697,10 @@ static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_trepack_q4_1_tiled(tensor, data, 0, size);break;+ case GGML_TYPE_Q4_K:+ repack_q4_K_tiled(tensor, data, 0, size);+ break;+case GGML_TYPE_Q8_0:repack_q8_0_tiled(tensor, data, 0, size);break;@@ -1324,6 +1713,10 @@ static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_trepack_mxfp4_tiled(tensor, data, 0, size);break;+ case GGML_TYPE_Q6_K:+ repack_q6_K_tiled(tensor, data, 0, size);+ break;+default:break;}@@ -1402,6 +1795,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,repack_tiled_q4_1(data, tensor, offset, size);break;+ case GGML_TYPE_Q4_K:+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor));+ repack_tiled_q4_K(data, tensor, offset, size);+ break;+case GGML_TYPE_Q8_0:GGML_ASSERT(offset == 0);GGML_ASSERT(offset + size <= ggml_nbytes(tensor));@@ -1420,6 +1819,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,repack_tiled_mxfp4(data, tensor, offset, size);break;+ case GGML_TYPE_Q6_K:+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor));+ repack_tiled_q6_K(data, tensor, offset, size);+ break;+default:memcpy(data, (const char *) tensor->data + offset, size);break;@@ -1525,6 +1930,10 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffrepack_tiled_q4_1(temp_buf.data(), tensor, offset, temp_size);break;+ case GGML_TYPE_Q4_K:+ repack_tiled_q4_K(temp_buf.data(), tensor, offset, temp_size);+ break;+case GGML_TYPE_Q8_0:repack_tiled_q8_0(temp_buf.data(), tensor, offset, temp_size);break;@@ -1537,6 +1946,10 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffrepack_tiled_mxfp4(temp_buf.data(), tensor, offset, temp_size);break;+ case GGML_TYPE_Q6_K:+ repack_tiled_q6_K(temp_buf.data(), tensor, offset, temp_size);+ break;+default:memcpy(temp_buf.data(), (const uint8_t *) tensor->data + offset, temp_size);break;@@ -1645,7 +2058,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffeint64_t ne1 = hex_round_up(t->ne[1], 32);int64_t ne2 = t->ne[2];int64_t ne3 = t->ne[3];- return ggml_row_size(t->type, ne0) * ne1 * ne2 * ne3;+ return ggml_hexagon_tiled_row_size(t->type, ne0) * ne1 * ne2 * ne3;}return ggml_nbytes(t);@@ -1806,7 +2219,7 @@ struct ggml_hexagon_opbatch {ne0 = hex_round_up(ne0, 32);ne1 = hex_round_up(ne1, 32);}- int64_t nb1 = is_repack ? ggml_row_size(t->type, ne0) : t->nb[1];+ int64_t nb1 = is_repack ? (int64_t) ggml_hexagon_tiled_row_size(t->type, ne0) : t->nb[1];int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2];int64_t nb3 = is_repack ? nb2 * t->ne[2] : t->nb[3];@@ -1855,7 +2268,7 @@ struct ggml_hexagon_opbatch {h.ne[3] = t->ne[3];h.nb[0] = t->nb[0];- h.nb[1] = ggml_row_size(t->type, h.ne[0]);+ h.nb[1] = ggml_hexagon_tiled_row_size(t->type, h.ne[0]);h.nb[2] = h.nb[1] * h.ne[1];h.nb[3] = h.nb[2] * h.ne[2];h.size = h.nb[3] * h.ne[3];@@ -3932,7 +4345,7 @@ static bool ggml_hexagon_precompute_hmx_mm_params(kparams->n_act_threads = act_threads_selected;kparams->tile_size = htp_mm_get_weight_tile_size(wtype);kparams->aligned_tile_size = aligned_tile_size;- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);kparams->vtcm_size = vtcm_size;kparams->vtcm_src0_size = 0;kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);@@ -3982,7 +4395,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(if (is_matmul_id) {kparams->kernel_type = (src1_nrows < (int) sess->n_threads) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);struct htp_mm_hvx_vtcm_layout L;uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;@@ -4011,7 +4424,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(} else {bool try_tiled = (k_align && opt_mm_select >= 2);if (try_tiled) {- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);if (src1_nrows < (int)sess->n_threads) {kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_BLOCK;} else {@@ -4052,7 +4465,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(// Flat HVX fallback{- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;struct htp_mm_hvx_vtcm_layout L;@@ -4444,7 +4857,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params({const int src1_nrows = ne11 * ne12 * ne13;- const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);const size_t src0_row_size = src0->nb[1];uint32_t best_n_prefetch = 16;@@ -4484,7 +4897,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params(kparams->n_weights = n_weights;} else {kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;- size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);+ size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);htp_mm_hvx_vtcm_layout_build(&L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads,@@ -4547,7 +4960,9 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * scase GGML_TYPE_Q8_0:case GGML_TYPE_IQ4_NL:case GGML_TYPE_MXFP4:- if (src0->ne[0] % 32) {+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q6_K:+ if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q4_K) ? QK_K : 32)) {return false;}@@ -4611,7 +5026,9 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_sessioncase GGML_TYPE_Q8_0:case GGML_TYPE_IQ4_NL:case GGML_TYPE_MXFP4:- if ((src0->ne[0] % 32)) {+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q6_K:+ if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q4_K) ? QK_K : 32)) {return false;}@@ -5347,8 +5764,8 @@ static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const strucreturn kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;}- if (!ggml_hexagon_is_repack_type(src0->type)) {- return false;+ if (!ggml_hexagon_is_repack_type(src0->type) || src0->type == GGML_TYPE_Q6_K) {+ return false; // Q6_K has no fused HVX kernel}return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;@@ -5378,7 +5795,7 @@ static bool is_mergeable_mul_mat(const ggml_tensor * t) {return ggml_hexagon_is_hmx_weight_type(src0->type);}- return ggml_hexagon_is_repack_type(src0->type);+ return ggml_hexagon_is_repack_type(src0->type) && src0->type != GGML_TYPE_Q6_K;}static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) {@@ -6767,6 +7184,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {"please update hexagon_type to match ggml_type");static_assert((unsigned int) HTP_TYPE_IQ4_NL == (unsigned int) GGML_TYPE_IQ4_NL,"please update hexagon_type to match ggml_type");+ static_assert((unsigned int) HTP_TYPE_Q4_K == (unsigned int) GGML_TYPE_Q4_K,+ "please update hexagon_type to match ggml_type");+ static_assert((unsigned int) HTP_TYPE_Q6_K == (unsigned int) GGML_TYPE_Q6_K,+ "please update hexagon_type to match ggml_type");const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE");const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH");diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.hindex 0011abba5a..d6d40586c5 100644--- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h+++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h@@ -506,6 +506,41 @@ static void dequantize_tiled_weight_to_fp16_task_q8_0(}}+// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q6_K.+// A k-group holds 4 k per row, the HMX tile holds 2, so each group is dealt into two tiles.+static void dequantize_tiled_weight_to_fp16_task_q6_k(+ const tiled_dequantize_state_t *state,+ uint32_t start_tile, uint32_t end_tile) {++ const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);+ const HVX_Vector i32 = Q6_Vb_vsplat_R(32);++ for (uint32_t t = start_tile; t < end_tile; t++) {+ const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size);+ __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;++ HVX_Vector v_sc = vptr[6];+ HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64);+ HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));+ HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2));++ #pragma unroll+ for (int g = 0; g < 8; g++) {+ const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16;++ HVX_Vector v_q = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32);+ HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q);+ HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4);++ hvx_vmem(dst_ptr + (2 * g + 0) * 64) =+ Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale));+ hvx_vmem(dst_ptr + (2 * g + 1) * 64) =+ Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale));+ }+ }+}+static __attribute__((noinline))void convert_f16_weight_to_fp16_tiles_task(const tiled_dequantize_state_t *state,@@ -803,15 +838,12 @@ static void transfer_output_chunk_fp16_to_fp32_col_chunk(HVX_Vector v = ((const HVX_Vector *) tile)[r1];HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one);- HVX_Vector *pv_out0 = (HVX_Vector *) (output_row_base + c + 0);- HVX_Vector *pv_out1 = (HVX_Vector *) (output_row_base + c + dst_stride);-HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));if (src2_row_base) {HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0);v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0);}- *pv_out0 = v_out0;+ hvx_vmemu(output_row_base + c + 0) = v_out0;if (r + 1 < n_rows) {HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));@@ -819,7 +851,7 @@ static void transfer_output_chunk_fp16_to_fp32_col_chunk(HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride);v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1);}- *pv_out1 = v_out1;+ hvx_vmemu(output_row_base + c + dst_stride) = v_out1;}}@@ -1366,12 +1398,9 @@ static void transfer_output_chunk_fp16_to_fp32_scattered(HVX_Vector v = ((const HVX_Vector *) tile)[r1];HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one);- HVX_Vector *pv_out0 = (HVX_Vector *) (output_row0 + c);- HVX_Vector *pv_out1 = output_row1 ? (HVX_Vector *) (output_row1 + c) : NULL;-- *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));- if (pv_out1) {- *pv_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));+ hvx_vmemu(output_row0 + c) = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));+ if (output_row1) {+ hvx_vmemu(output_row1 + c) = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));}}}diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.hindex 869b19b8c2..98a5f6d5c1 100644--- a/ggml/src/ggml-hexagon/htp/htp-ops.h+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h@@ -22,6 +22,8 @@ enum htp_data_type {HTP_TYPE_Q4_0 = 2,HTP_TYPE_Q4_1 = 3,HTP_TYPE_Q8_0 = 8,+ HTP_TYPE_Q4_K = 12,+ HTP_TYPE_Q6_K = 14,HTP_TYPE_IQ4_NL = 20,HTP_TYPE_I32 = 26,HTP_TYPE_I64 = 27,diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.hindex 328a831189..5c1372cf1b 100644--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h@@ -744,6 +744,143 @@ static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float}}+static void flat_vec_dot_q6_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {+ const uint8_t * restrict tile_ptr = vx;+ const uint8_t * restrict y_q = vy;++ HVX_Vector v_sum_float = Q6_V_vzero();+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);++ static const uint8_t __attribute__((aligned(128))) repl[128] = {+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ };+ HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl;++ const uint32_t quants_size = hex_round_up(n, 128);+ const __fp16 * restrict y_scales = (const __fp16 *) (y_q + quants_size);++ uint32_t n_k_tiles = n / 32;+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);++ uint32_t block_idx = kt / 4;+ uint32_t sub_idx = kt % 4;++ HVX_Vector vx_i8 = * (const HVX_Vector *) (y_q + block_idx * 128);+ HVX_Vector v_act_raw = Q6_V_vror_VR(vx_i8, sub_idx * 32);++ HVX_Vector v_act_rep[8];+ v_act_rep[0] = Q6_V_vdelta_VV(v_act_raw, v_repl_ctrl);+ v_act_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 4), v_repl_ctrl);+ v_act_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 8), v_repl_ctrl);+ v_act_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 12), v_repl_ctrl);+ v_act_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 16), v_repl_ctrl);+ v_act_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 20), v_repl_ctrl);+ v_act_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 24), v_repl_ctrl);+ v_act_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 28), v_repl_ctrl);++ HVX_VectorPair v_sums = accum_q6_k_32x1(vptr, v_act_rep, i32);++ __fp16 scale_a_val = y_scales[kt];+ HVX_Vector v_scale_a = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a_val));++ v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q6_k_32x1(v_sums, vptr[6], v_scale_a));+ }++ if (sz) {+ hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));+ } else {+ hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);+ }+}++static void flat_vec_dot_q6_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {+ const uint8_t * restrict tile_ptr = vx;+ const uint8_t * restrict y0_q = vy0;+ const uint8_t * restrict y1_q = vy1;++ HVX_Vector v_sum_float_c0 = Q6_V_vzero();+ HVX_Vector v_sum_float_c1 = Q6_V_vzero();+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);++ static const uint8_t __attribute__((aligned(128))) repl[128] = {+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,+ };+ HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl;++ const uint32_t quants_size = hex_round_up(n, 128);+ const __fp16 * restrict y0_scales = (const __fp16 *) (y0_q + quants_size);+ const __fp16 * restrict y1_scales = (const __fp16 *) (y1_q + quants_size);++ uint32_t n_k_tiles = n / 32;+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);++ uint32_t block_idx = kt / 4;+ uint32_t sub_idx = kt % 4;++ HVX_Vector vx0_i8 = * (const HVX_Vector *) (y0_q + block_idx * 128);+ HVX_Vector vx1_i8 = * (const HVX_Vector *) (y1_q + block_idx * 128);+ HVX_Vector v_act0_raw = Q6_V_vror_VR(vx0_i8, sub_idx * 32);+ HVX_Vector v_act1_raw = Q6_V_vror_VR(vx1_i8, sub_idx * 32);++ HVX_Vector v_act0_rep[8];+ HVX_Vector v_act1_rep[8];+ v_act0_rep[0] = Q6_V_vdelta_VV(v_act0_raw, v_repl_ctrl);+ v_act0_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 4), v_repl_ctrl);+ v_act0_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 8), v_repl_ctrl);+ v_act0_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 12), v_repl_ctrl);+ v_act0_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 16), v_repl_ctrl);+ v_act0_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 20), v_repl_ctrl);+ v_act0_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 24), v_repl_ctrl);+ v_act0_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 28), v_repl_ctrl);+ v_act1_rep[0] = Q6_V_vdelta_VV(v_act1_raw, v_repl_ctrl);+ v_act1_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 4), v_repl_ctrl);+ v_act1_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 8), v_repl_ctrl);+ v_act1_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 12), v_repl_ctrl);+ v_act1_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 16), v_repl_ctrl);+ v_act1_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 20), v_repl_ctrl);+ v_act1_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 24), v_repl_ctrl);+ v_act1_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 28), v_repl_ctrl);++ HVX_VectorPair v_sums0, v_sums1;+ accum_q6_k_32x2(vptr, v_act0_rep, v_act1_rep, i32, &v_sums0, &v_sums1);++ __fp16 scale_a0_val = y0_scales[kt];+ __fp16 scale_a1_val = y1_scales[kt];+ HVX_Vector v_scale_a0 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a0_val));+ HVX_Vector v_scale_a1 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a1_val));++ v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q6_k_32x1(v_sums0, vptr[6], v_scale_a0));+ v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q6_k_32x1(v_sums1, vptr[6], v_scale_a1));+ }++ if (sz0) {+ hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));+ } else {+ hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);+ }+ if (sz1) {+ hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));+ } else {+ hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);+ }+}+static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {const uint8_t * restrict tile_ptr = vx;const uint8_t * restrict y_q = vy;diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.hindex 40b65aa3b5..c889538ac8 100644--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h@@ -378,6 +378,74 @@ static inline HVX_VectorPair accum_q8_0_32x2(return Q6_W_vcombine_VV(v_sum1, v_sum0);}+// Q6_K weights are stored unsigned (0..63), see HTP_MM_WEIGHT_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32)+static inline HVX_Vector unpack_q6_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_0f, HVX_Vector mask_03, HVX_Vector i32) {+ HVX_Vector v_lo = (g & 1) ? Q6_Vub_vlsr_VubR(vptr[g >> 1], 4) : Q6_V_vand_VV(vptr[g >> 1], mask_0f);+ HVX_Vector v_hi = (g & 3) ? Q6_Vub_vlsr_VubR(vptr[4 + (g >> 2)], 2 * (g & 3)) : vptr[4 + (g >> 2)];+ HVX_Vector v_q = Q6_V_vor_VV(v_lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_hi, mask_03), 4));+ return Q6_Vb_vsub_VbVb(v_q, i32);+}++// k 0..15 and k 16..31 of a Q6_K tile have different scales: lo half of the pair sums k 0..15, hi half sums k 16..31+static inline HVX_VectorPair accum_q6_k_32x1(+ const HVX_Vector * restrict vptr,+ const HVX_Vector * restrict v_act,+ HVX_Vector i32+) {+ HVX_Vector v_sum_lo = Q6_V_vzero();+ HVX_Vector v_sum_hi = Q6_V_vzero();+ HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);+ HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);++ #pragma unroll+ for (int g = 0; g < 4; g++) {+ HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32);+ HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32);+ v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]);+ v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]);+ }++ return Q6_W_vcombine_VV(v_sum_hi, v_sum_lo);+}++static inline void accum_q6_k_32x2(+ const HVX_Vector * restrict vptr,+ const HVX_Vector * restrict v_act0,+ const HVX_Vector * restrict v_act1,+ HVX_Vector i32,+ HVX_VectorPair * v_sums0,+ HVX_VectorPair * v_sums1+) {+ HVX_Vector v_sum0_lo = Q6_V_vzero();+ HVX_Vector v_sum0_hi = Q6_V_vzero();+ HVX_Vector v_sum1_lo = Q6_V_vzero();+ HVX_Vector v_sum1_hi = Q6_V_vzero();+ HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);+ HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);++ #pragma unroll+ for (int g = 0; g < 4; g++) {+ HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32);+ HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32);+ v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]);+ v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]);+ v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]);+ v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]);+ }++ *v_sums0 = Q6_W_vcombine_VV(v_sum0_hi, v_sum0_lo);+ *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);+}++// scale the two half sums with the per-row tile scales (v_scale_w = vptr[6]) and the activation scale+static inline HVX_Vector scale_q6_k_32x1(HVX_VectorPair v_sums, HVX_Vector v_scale_w, HVX_Vector v_scale_a) {+ HVX_Vector v_scale_lo = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a);+ HVX_Vector v_scale_hi = hvx_vec_mul_f16_f16_to_f32_lower32(Q6_V_vror_VR(v_scale_w, 64), v_scale_a);+ HVX_Vector v_lo = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums)), v_scale_lo);+ HVX_Vector v_hi = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums)), v_scale_hi);+ return hvx_vec_add_f32_f32(v_lo, v_hi);+}+static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {const uint8_t * restrict tile_ptr = vx;const uint8_t * restrict y_q = vy;@@ -785,6 +853,63 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float}}+static void tiled_vec_dot_q6_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {+ const uint8_t * restrict tile_ptr = vx;+ const uint8_t * restrict y_q = vy;++ HVX_Vector v_sum_float = Q6_V_vzero();+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);++ uint32_t n_k_tiles = n / 32;+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);+ const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152);++ HVX_VectorPair v_sums = accum_q6_k_32x1(vptr, v_act, i32);+ v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q6_k_32x1(v_sums, vptr[6], v_act[8]));+ }++ if (sz) {+ hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));+ } else {+ hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);+ }+}++static void tiled_vec_dot_q6_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {+ const uint8_t * restrict tile_ptr = vx;+ const uint8_t * restrict y0_q = vy0;+ const uint8_t * restrict y1_q = vy1;++ HVX_Vector v_sum_float_c0 = Q6_V_vzero();+ HVX_Vector v_sum_float_c1 = Q6_V_vzero();+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);++ uint32_t n_k_tiles = n / 32;+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);+ const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152);+ const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152);++ HVX_VectorPair v_sums0, v_sums1;+ accum_q6_k_32x2(vptr, v_act0, v_act1, i32, &v_sums0, &v_sums1);++ v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q6_k_32x1(v_sums0, vptr[6], v_act0[8]));+ v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q6_k_32x1(v_sums1, vptr[6], v_act1[8]));+ }++ if (sz0) {+ hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));+ } else {+ hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);+ }+ if (sz1) {+ hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));+ } else {+ hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);+ }+}+static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {const uint8_t * restrict tile_ptr = vx;const uint8_t * restrict y_q = vy;diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.cindex 1b597dcd9f..e16cfdcbe2 100644--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c@@ -325,8 +325,9 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) {}}-#include "hmx-mm-kernels-tiled.h"+// hvx kernels first: the HMX Q6_K dequantizer reuses unpack_q6_k_group from there#include "hvx-mm-kernels-tiled.h"+#include "hmx-mm-kernels-tiled.h"#include "hvx-mm-kernels-flat.h"// Specialized repacked matmul macros@@ -637,12 +638,14 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, vMATMUL_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1)MATMUL_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1)MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1)+MATMUL_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1)MATMUL_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)MATMUL_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)MATMUL_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1)MATMUL_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1)MATMUL_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1)+MATMUL_2D_REPACKED_IMPL(q6_k_flat, 896, flat_vec_dot_q6_k_32x2, flat_vec_dot_q6_k_32x1)MATMUL_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1)MATMUL_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1)@@ -737,12 +740,14 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, voMATVEC_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x1)MATVEC_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x1)MATVEC_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x1)+MATVEC_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x1)MATVEC_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x1)MATVEC_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x1)MATVEC_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x1)MATVEC_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x1)MATVEC_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x1)+MATVEC_2D_REPACKED_IMPL(q6_k_flat, 896, flat_vec_dot_q6_k_32x1)MATVEC_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x1)MATVEC_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x1)@@ -1339,6 +1344,7 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_typemmctx->vec_dot_32x1 = tiled_vec_dot_q4_0_32x1;return 0;case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K:mmctx->type = "q4_1_tiled-f32";mmctx->vec_dot_32x1 = tiled_vec_dot_q4_1_32x1;return 0;@@ -1346,6 +1352,10 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_typemmctx->type = "q8_0_tiled-f32";mmctx->vec_dot_32x1 = tiled_vec_dot_q8_0_32x1;return 0;+ case HTP_TYPE_Q6_K:+ mmctx->type = "q6_k_tiled-f32";+ mmctx->vec_dot_32x1 = tiled_vec_dot_q6_k_32x1;+ return 0;case HTP_TYPE_IQ4_NL:mmctx->type = "iq4nl_tiled-f32";mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1;@@ -1395,7 +1405,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 ||src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL ||- src0->type == HTP_TYPE_MXFP4);+ src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q6_K ||+ src0->type == HTP_TYPE_Q4_K);// Compute src0_nrows_per_threadmmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);@@ -1419,8 +1430,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {if (is_repacked) {switch (src0->type) {case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_2d_repacked_q4_0; break;- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_2d_repacked_q4_1; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_2d_repacked_q4_1; break;case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_2d_repacked_q8_0; break;+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_2d_repacked_q6_k; break;case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_2d_repacked_iq4nl; break;case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_2d_repacked_mxfp4; break;default: return HTP_STATUS_NO_SUPPORT;@@ -1432,8 +1445,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {if (is_repacked) {switch (src0->type) {case HTP_TYPE_Q4_0: matmul_job_func = hvx_mv_2d_repacked_q4_0; break;- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mv_2d_repacked_q4_1; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mv_2d_repacked_q4_1; break;case HTP_TYPE_Q8_0: matmul_job_func = hvx_mv_2d_repacked_q8_0; break;+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mv_2d_repacked_q6_k; break;case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mv_2d_repacked_iq4nl; break;case HTP_TYPE_MXFP4: matmul_job_func = hvx_mv_2d_repacked_mxfp4; break;default: return HTP_STATUS_NO_SUPPORT;@@ -1505,14 +1520,16 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: {n_quant_tasks = MIN(src1_nrows, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);if (src1_nrows > 1) {switch (src0->type) {case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_2d_repacked_q4_0_flat; break;- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_2d_repacked_q4_1_flat; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_2d_repacked_q4_1_flat; break;case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_2d_repacked_q8_0_flat; break;+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_2d_repacked_q6_k_flat; break;case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_2d_repacked_iq4nl_flat; break;case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_2d_repacked_mxfp4_flat; break;default: return HTP_STATUS_NO_SUPPORT;@@ -1520,8 +1537,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {} else {switch (src0->type) {case HTP_TYPE_Q4_0: matmul_job_func = hvx_mv_2d_repacked_q4_0_flat; break;- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mv_2d_repacked_q4_1_flat; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mv_2d_repacked_q4_1_flat; break;case HTP_TYPE_Q8_0: matmul_job_func = hvx_mv_2d_repacked_q8_0_flat; break;+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mv_2d_repacked_q6_k_flat; break;case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mv_2d_repacked_iq4nl_flat; break;case HTP_TYPE_MXFP4: matmul_job_func = hvx_mv_2d_repacked_mxfp4_flat; break;default: return HTP_STATUS_NO_SUPPORT;@@ -1543,7 +1562,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {if (src1_nrows < octx->n_threads) {n_quant_tasks = MIN(total_nb, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {uint32_t ib_first = (total_nb * ith) / n_quant_tasks;uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;@@ -1554,9 +1573,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {}} else {n_quant_tasks = MIN(src1_nrows, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;}- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);break;}@@ -1742,6 +1761,7 @@ DEQUANTIZE_WORKER_LOOP_IMPL(q4_1)DEQUANTIZE_WORKER_LOOP_IMPL(iq4_nl)DEQUANTIZE_WORKER_LOOP_IMPL(mxfp4)DEQUANTIZE_WORKER_LOOP_IMPL(q8_0)+DEQUANTIZE_WORKER_LOOP_IMPL(q6_k)static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) {tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data;@@ -2476,9 +2496,11 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,switch (weight_type) {case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break;case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break;- case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break;case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break;+ case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break;case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break;case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break;default:@@ -2732,9 +2754,11 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_kswitch (weight_type) {case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break;case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break;- case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break;case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break;+ case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break;case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break;case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break;default:@@ -3324,9 +3348,11 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,switch (weight_type) {case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break;case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break;- case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break;case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break;+ case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break;case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break;case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break;default:@@ -3626,7 +3652,7 @@ static int hvx_mm_matmul_id(uint32_t n_quant_tasks = 1;if (src1_nrows < octx->n_threads) {n_quant_tasks = MIN(total_nb, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {uint32_t ib_first = (total_nb * ith) / n_quant_tasks;uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;@@ -3637,9 +3663,9 @@ static int hvx_mm_matmul_id(}} else {n_quant_tasks = MIN(src1_nrows, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;}- size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);struct htp_mm_hvx_vtcm_layout L;htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads,@@ -3773,7 +3799,7 @@ static int hvx_mm_matmul_id_nx(uint32_t n_quant_tasks = 1;if (src1_nrows < octx->n_threads) {n_quant_tasks = MIN(total_nb, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {uint32_t ib_first = (total_nb * ith) / n_quant_tasks;uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;@@ -3784,9 +3810,9 @@ static int hvx_mm_matmul_id_nx(}} else {n_quant_tasks = MIN(src1_nrows, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;}- size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);+ size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);struct htp_mm_hvx_vtcm_layout L;htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads,@@ -4129,7 +4155,7 @@ int op_matmul_nx(struct htp_ops_context * octx) {bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 ||src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL ||- src0->type == HTP_TYPE_MXFP4);+ src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q4_K);struct htp_mm_context mmctx_struct = {0};struct htp_mm_context * mmctx = &mmctx_struct;@@ -4153,10 +4179,10 @@ int op_matmul_nx(struct htp_ops_context * octx) {uint32_t n_quant_tasks = 1;if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {n_quant_tasks = MIN(src1_nrows, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;} else if (src1_nrows < octx->n_threads) {n_quant_tasks = MIN(total_nb, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {uint32_t ib_first = (total_nb * ith) / n_quant_tasks;uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;@@ -4167,14 +4193,14 @@ int op_matmul_nx(struct htp_ops_context * octx) {}} else {n_quant_tasks = MIN(src1_nrows, octx->n_threads);- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;}size_t src1_row_size;if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(act->ne[0]) : htp_mm_q8_0_flat_row_size(act->ne[0]);+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(act->ne[0]) : htp_mm_q8_0_flat_row_size(act->ne[0]);} else {- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);}struct htp_mm_hvx_vtcm_layout L;@@ -4219,7 +4245,8 @@ int op_matmul_nx(struct htp_ops_context * octx) {if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {switch (src0->type) {case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0_flat; break;- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1_flat; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1_flat; break;case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0_flat; break;case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl_flat; break;case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4_flat; break;@@ -4228,7 +4255,8 @@ int op_matmul_nx(struct htp_ops_context * octx) {} else {switch (src0->type) {case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0; break;- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break;+ case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break;case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0; break;case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl; break;case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4; break;diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.hindex 2dbcb0c2e5..1df8c2933c 100644--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h@@ -25,6 +25,11 @@ extern "C" {#define HTP_MM_WEIGHT_TILE_SIZE_Q8_0 1088#define HTP_MM_WEIGHT_TILE_SIZE_IQ4_NL 576#define HTP_MM_WEIGHT_TILE_SIZE_MXFP4 544+// Q6_K native 6-bit tile (32 rows x 32 k), vrmpy-ready: byte 4*row+b of a vector holds k = 4*group+b+// vectors 0..3: low nibbles, vector i holds group 2i (low nibble) and group 2i+1 (high nibble)+// vectors 4..5: high 2 bits, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6+// vector 6: fp16 scales per row, d * scales[]: k 0..15 in lanes 0..31, k 16..31 in lanes 32..63+#define HTP_MM_WEIGHT_TILE_SIZE_Q6_K 896// --- Weight Repacked Aligned Tile Sizes ---#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0 640@@ -32,6 +37,7 @@ extern "C" {#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0 1152#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_IQ4_NL 640#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4 640+#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K 896// --- Activation Tiled Block Sizes (including padding) ---#define HTP_MM_ACT_TILE_SIZE_Q8_0 1152@@ -195,9 +201,12 @@ static inline uint32_t htp_mm_get_weight_tile_size(int weight_type) {case HTP_TYPE_IQ4_NL:return HTP_MM_WEIGHT_TILE_SIZE_Q4_0;case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K:return HTP_MM_WEIGHT_TILE_SIZE_Q4_1;case HTP_TYPE_Q8_0:return HTP_MM_WEIGHT_TILE_SIZE_Q8_0;+ case HTP_TYPE_Q6_K:+ return HTP_MM_WEIGHT_TILE_SIZE_Q6_K;case HTP_TYPE_MXFP4:return HTP_MM_WEIGHT_TILE_SIZE_MXFP4;default:@@ -211,9 +220,12 @@ static inline uint32_t htp_mm_get_weight_aligned_tile_size(int weight_type) {case HTP_TYPE_IQ4_NL:return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0;case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K:return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_1;case HTP_TYPE_Q8_0:return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0;+ case HTP_TYPE_Q6_K:+ return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K;case HTP_TYPE_MXFP4:return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4;default:@@ -254,7 +266,9 @@ static inline size_t htp_mm_get_tiled_row_stride(int weight_type, uint32_t k) {case HTP_TYPE_Q4_0:case HTP_TYPE_IQ4_NL:case HTP_TYPE_Q4_1:+ case HTP_TYPE_Q4_K:case HTP_TYPE_Q8_0:+ case HTP_TYPE_Q6_K:case HTP_TYPE_MXFP4:return (size_t) nb * htp_mm_get_weight_tile_size(weight_type);case HTP_TYPE_F16:@@ -484,7 +498,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 ||wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL ||- wtype == HTP_TYPE_MXFP4);+ wtype == HTP_TYPE_MXFP4 || wtype == HTP_TYPE_Q6_K ||+ wtype == HTP_TYPE_Q4_K);if (is_fused_nx) {const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);@@ -502,8 +517,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);}- size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);- size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);+ size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);size_t act_sz = (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT)? hex_round_up(flat_act_row_size * src1_nrows, 128)@@ -516,8 +531,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(dst_sz = quant_scratch_size;} else if (is_matmul_id) {const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128);- const size_t src1_row_size_tiled = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10)- : htp_mm_q8_0_tiled_row_size(ne10);+ const size_t src1_row_size_tiled = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10)+ : htp_mm_q8_0_tiled_row_size(ne10);size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);src1_sz = htp_mm_round_up(src1_row_size_tiled * src1_nrows, 256);@@ -562,7 +577,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(}case HTP_MM_KERNEL_HVX_QUANT_BLOCK:case HTP_MM_KERNEL_HVX_QUANT_ROW: {- size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);+ size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256);@@ -584,7 +599,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(break;}case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: {- size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);+ size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256);