FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(kernel): add an iq1_m mmq kernel for moe steps above 8 tokens (#835)
Chen Chengd2977fbc13c9
diff --git a/python/freetoken/kernel/csrc/ggml/VERSION b/python/freetoken/kernel/csrc/ggml/VERSIONindex c4a9671d..114ad366 100644--- a/python/freetoken/kernel/csrc/ggml/VERSION+++ b/python/freetoken/kernel/csrc/ggml/VERSION@@ -12,5 +12,6 @@ Local changes, kept in freetoken.patch (already applied):- common.cuh: the native fp4 MMA (BLACKWELL_MMA_AVAILABLE, blackwell_mma_available) is off unless GGML_CUDA_FP4_MMA is defined; it needs an sm_120a/sm_120f build.- mmvq.cu, common.cuh: the NVFP4 scale fusion (x_scale, gate_scale) reads strided scale vectors, e.g. columns of a per-expert scale bank.- getrows.cu, dequantize.cuh: get_rows reads NVFP4 rows, e.g. an NVFP4 token_embd.+- mmq.cu/.cuh, mmq-load-tiles.cuh, template-instances/mmq-instance-iq1_m.cu: MMQ runs IQ1_M, which llama.cpp has no MMQ kernel for. Its values times 8 are int8, so it loads into IQ2_XS's tile layout and takes IQ2_XS's configs.To update: copy the same files from a newer llama.cpp, apply freetoken.patch, and update this file.diff --git a/python/freetoken/kernel/csrc/ggml/freetoken.patch b/python/freetoken/kernel/csrc/ggml/freetoken.patchindex e968a078..e104a451 100644--- a/python/freetoken/kernel/csrc/ggml/freetoken.patch+++ b/python/freetoken/kernel/csrc/ggml/freetoken.patch@@ -122,9 +122,98 @@++void ggml_cuda_launch_mm_ids_tile_map(+ const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream);+--- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh++++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh+@@ -1095,6 +1095,76 @@+ }+ }+++// FreeToken: IQ1_M's delta changes every 8 values, so it cannot use IQ1_S's per-32 dm. 8 * (q + delta) is an integer++// (8q - 7, or 8q - 9 with the shift bit), so the tile holds those int8 values and d * scale / 8 per 16, like IQ2_XS.++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_m(++ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {++ constexpr int warp_size = ggml_cuda_get_physical_warp_size();++ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);++ int * x_qs = (int *) x_tile;++ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);++#else++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ1_M, I);++ int * x_qs = (int *) x_tile;++ float * x_df = (float *) (x_qs + txs.qs);++#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_M);++ constexpr int nrows = warp_size / threads_per_row;++ const int kqsx = threadIdx.x % threads_per_row;++++#pragma unroll++ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {++ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++++ if (fallback) {++ i = min(i, i_max);++ }++++ const block_iq1_m * bxi = (const block_iq1_m *) x + kbx0 + i*stride;++++ const int qs_packed = get_int_b4(bxi->qs, kqsx);++ const uint8_t * qs = (const uint8_t *) &qs_packed;++++ const int qh = bxi->qh[2*kqsx + 0] | (bxi->qh[2*kqsx + 1] << 8);++++ #pragma unroll++ for (int l = 0; l < QR1_M/2; ++l) {++ const int qhl = qh >> (4*l);++ const int grid = iq1s_grid_gpu[qs[l] | ((qhl & 0x07) << 8)];++ const int off = qhl & 0x08 ? 0x09090909 : 0x07070707;++++ const int grid0 = __vsub4(((grid >> 0) & 0x0F0F0F0F) << 3, off);++ const int grid1 = __vsub4(((grid >> 4) & 0x0F0F0F0F) << 3, off);++++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;++ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;++#else++ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;++ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;++#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ }++++ const uint16_t * sc = (const uint16_t *) bxi->scales;++ iq1m_scale_t scale;++ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);++ const float d = __half2float(scale.f16) / 8;++ const int ls = sc[kqsx/2] >> (6*(kqsx%2));++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ x_df[i*sram_stride + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);++ x_df[i*sram_stride + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);++#else++ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);++ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);++#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ }++}+++ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();--- a/ggml/src/ggml-cuda/mmq.cu+++ b/ggml/src/ggml-cuda/mmq.cu-@@ -132,9 +132,11 @@+@@ -48,6 +48,9 @@+ case GGML_TYPE_IQ1_S:+ mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);+ break;++ case GGML_TYPE_IQ1_M:++ mul_mat_q_case<GGML_TYPE_IQ1_M, GGML_PREC_Q8>(ctx, args, stream);++ break;+ case GGML_TYPE_IQ2_XXS:+ mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream);+ break;+@@ -132,9 +135,11 @@}void ggml_cuda_mul_mat_q(@@ -139,7 +228,7 @@GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.GGML_TENSOR_BINARY_OP_LOCALS;-@@ -156,7 +158,9 @@+@@ -156,7 +161,9 @@const char * src0_d = (const char *) src0->data;const float * src1_d = (const float *) src1->data;@@ -150,7 +239,7 @@// If src0 is a temporary compute buffer, clear any potential padding.if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {-@@ -183,6 +187,7 @@+@@ -183,6 +190,7 @@const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;@@ -158,7 +247,7 @@const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ;-@@ -192,7 +197,8 @@+@@ -192,7 +200,8 @@int64_t ncols_opt = ne11;if (ids) {const int64_t n_expert_used = ids->ne[0];@@ -168,7 +257,7 @@// Each expert only sees ne12*n_expert_used/ne02 tokens on average.// On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.-@@ -249,6 +255,12 @@+@@ -249,6 +258,12 @@quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,ne11, ne12, ne13, stream);@@ -181,7 +270,7 @@} else {quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,ne11, ne12, ne13, stream);-@@ -299,6 +311,17 @@+@@ -299,6 +314,17 @@CUDA_CHECK(cudaGetLastError());}@@ -199,7 +288,7 @@const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());-@@ -325,9 +348,21 @@+@@ -325,9 +351,21 @@quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);}@@ -221,7 +310,7 @@} else {quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);-@@ -347,7 +382,7 @@+@@ -347,7 +385,7 @@ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,ne02, ne02, s02, s12, s2,ne03, ne13, s03, s13, s3,@@ -230,6 +319,14 @@ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);}+@@ -375,6 +413,7 @@+ case GGML_TYPE_Q6_K:+ // -------------------------------------------------+ case GGML_TYPE_IQ1_S:++ case GGML_TYPE_IQ1_M:+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:--- a/ggml/src/ggml-cuda/mmq.cuh+++ b/ggml/src/ggml-cuda/mmq.cuh@@ -10,6 +10,17 @@@@ -250,7 +347,52 @@typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,-@@ -429,8 +440,8 @@+@@ -88,6 +99,7 @@+ case GGML_TYPE_IQ2_S:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ3_S:++ case GGML_TYPE_IQ1_M:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_IQ1_S:+ return MMQ_Q8_1_DS_LAYOUT_DS4;+@@ -227,6 +239,14 @@+ #undef CASE++ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {++ // FreeToken: IQ1_M fills the same tile as IQ2_XS (int8 values, one scale per 16), so it takes IQ2_XS's configs++ if (type == GGML_TYPE_IQ1_M) {++ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, cc, prec_src1);++ if (config.type == GGML_TYPE_IQ2_XS) {++ config.type = GGML_TYPE_IQ1_M;++ }++ return config;++ }+ if (GGML_CUDA_CC_IS_AMD(cc)) {+ if (GGML_CUDA_CC_IS_GCN(cc)) {+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);+@@ -262,6 +282,13 @@+ }++ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {++ if (type == GGML_TYPE_IQ1_M) {++ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, prec_src1);++ if (config.type == GGML_TYPE_IQ2_XS) {++ config.type = GGML_TYPE_IQ1_M;++ }++ return config;++ }+ #ifdef GGML_USE_HIP+ #ifdef GCN+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);+@@ -409,6 +436,7 @@+ case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;++ case GGML_TYPE_IQ1_M: return MMQ_DP4A_TXS_Q8_0_16;+ case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;+ default: return tile_x_sizes{0, 0, 0};+@@ -429,8 +457,8 @@#include "mmq-load-tiles.cuh"#include "mmq-vec-dot.cuh"@@ -261,7 +403,7 @@const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {constexpr int warp_size = ggml_cuda_get_physical_warp_size();constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;-@@ -456,21 +467,21 @@+@@ -456,21 +484,21 @@if constexpr (type == GGML_TYPE_NVFP4) {if (y_scale_used) {@@ -288,7 +430,7 @@const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-@@ -506,12 +517,12 @@+@@ -506,12 +534,12 @@if constexpr (type == GGML_TYPE_NVFP4) {if (y_scale_used) {@@ -304,7 +446,33 @@GGML_UNUSED(y_scale_used);}}-@@ -869,10 +880,10 @@+@@ -617,6 +645,12 @@+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);++ case GGML_TYPE_IQ1_M:++ return ggml_cuda_mmq_util_funcs(++ VDR_IQ2_XS_Q8_1_MMQ,++ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,++ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,++ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_XXS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_XXS_Q8_1_MMQ,+@@ -787,6 +821,12 @@+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);++ case GGML_TYPE_IQ1_M:++ return ggml_cuda_mmq_util_funcs(++ -1,++ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,++ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,++ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_XXS:+ return ggml_cuda_mmq_util_funcs(+ -1,+@@ -869,10 +909,10 @@// ---------------------------------------------------------------------------------------------@@ -317,7 +485,7 @@const float * __restrict__ y_scale,const int stride_row_x, const int ncols_y, const int stride_col_dst,const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {-@@ -942,19 +953,32 @@+@@ -942,19 +982,32 @@if (fixup) {write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);@@ -353,7 +521,7 @@const float * __restrict__ y_scale,const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,-@@ -993,9 +1017,10 @@+@@ -993,9 +1046,10 @@if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);const int wt = tmp2.x;@@ -366,7 +534,7 @@// Defaults for regular matrix multiplication:int col_low = 0;-@@ -1076,14 +1101,15 @@+@@ -1076,14 +1130,15 @@while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {int tmp = fastdiv(kbc, blocks_per_ne00);uint2 tmp2 = fast_div_modulo(tmp, ntx);@@ -384,7 +552,7 @@// Defaults for regular matrix multiplication:int col_low = 0;-@@ -1165,14 +1191,15 @@+@@ -1165,14 +1220,15 @@int tmp = fastdiv(kbc, blocks_per_ne00);uint2 tmp2 = fast_div_modulo(tmp, ntx);@@ -402,7 +570,7 @@// Defaults for regular matrix multiplication:int col_low = 0;-@@ -1375,14 +1402,26 @@+@@ -1375,14 +1431,26 @@}}@@ -430,7 +598,7 @@};static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {-@@ -1406,12 +1445,11 @@+@@ -1406,12 +1474,11 @@const dim3 block_dims(warp_size, nwarps, 1);@@ -447,7 +615,7 @@const dim3 block_nums_xy_tiling(nty, ntx, ntzw);GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);-@@ -1421,18 +1459,71 @@+@@ -1421,18 +1488,71 @@const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);const uint3 ntx_fd = init_fastdiv_values(ntx);@@ -523,7 +691,7 @@return;}-@@ -1441,7 +1532,8 @@+@@ -1441,7 +1561,8 @@const int ntiles_dst = ntx * nty * ntzw;const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);@@ -533,7 +701,7 @@GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.-@@ -1453,25 +1545,7 @@+@@ -1453,25 +1574,7 @@tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);}@@ -560,7 +728,15 @@}template <ggml_type type, bool fallback, ggml_prec prec_src1>-@@ -1579,7 +1653,9 @@+@@ -1564,6 +1667,7 @@+ extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);+ // -----------------------------------------+ extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);++extern DECL_MMQ_CASE(GGML_TYPE_IQ1_M);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);+@@ -1579,7 +1683,9 @@// -------------------------------------------------------------------------------------------------------------------------@@ -1087,3 +1263,11 @@const int32_t * ids_src1_inv,void * vy,ggml_type type_src0,+--- /dev/null++++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_m.cu+@@ -0,0 +1,5 @@++// FreeToken: not generated, llama.cpp has no IQ1_M MMQ.++++#include "../mmq.cuh"++++DECL_MMQ_CASE(GGML_TYPE_IQ1_M);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuhindex 017b93b5..4dd92170 100644--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh@@ -1095,6 +1095,76 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_}}+// FreeToken: IQ1_M's delta changes every 8 values, so it cannot use IQ1_S's per-32 dm. 8 * (q + delta) is an integer+// (8q - 7, or 8q - 9 with the shift bit), so the tile holds those int8 values and d * scale / 8 per 16, like IQ2_XS.+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_m(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ1_M, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_M);+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq1_m * bxi = (const block_iq1_m *) x + kbx0 + i*stride;++ const int qs_packed = get_int_b4(bxi->qs, kqsx);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bxi->qh[2*kqsx + 0] | (bxi->qh[2*kqsx + 1] << 8);++ #pragma unroll+ for (int l = 0; l < QR1_M/2; ++l) {+ const int qhl = qh >> (4*l);+ const int grid = iq1s_grid_gpu[qs[l] | ((qhl & 0x07) << 8)];+ const int off = qhl & 0x08 ? 0x09090909 : 0x07070707;++ const int grid0 = __vsub4(((grid >> 0) & 0x0F0F0F0F) << 3, off);+ const int grid1 = __vsub4(((grid >> 4) & 0x0F0F0F0F) << 3, off);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const uint16_t * sc = (const uint16_t *) bxi->scales;+ iq1m_scale_t scale;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);+ const float d = __half2float(scale.f16) / 8;+ const int ls = sc[kqsx/2] >> (6*(kqsx%2));+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);+ x_df[i*sram_stride + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);+#else+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {constexpr int warp_size = ggml_cuda_get_physical_warp_size();diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuindex f754837d..b1c424de 100644--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu@@ -48,6 +48,9 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, concase GGML_TYPE_IQ1_S:mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);break;+ case GGML_TYPE_IQ1_M:+ mul_mat_q_case<GGML_TYPE_IQ1_M, GGML_PREC_Q8>(ctx, args, stream);+ break;case GGML_TYPE_IQ2_XXS:mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream);break;@@ -410,6 +413,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_tcase GGML_TYPE_Q6_K:// -------------------------------------------------case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ1_M:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:case GGML_TYPE_IQ2_S:diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuhindex 596a54ea..154a628a 100644--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh@@ -99,6 +99,7 @@ static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {case GGML_TYPE_IQ2_S:case GGML_TYPE_IQ3_XXS:case GGML_TYPE_IQ3_S:+ case GGML_TYPE_IQ1_M:return MMQ_Q8_1_DS_LAYOUT_D4;case GGML_TYPE_IQ1_S:return MMQ_Q8_1_DS_LAYOUT_DS4;@@ -238,6 +239,14 @@ struct ggml_cuda_mmq_config {#undef CASEstatic __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ // FreeToken: IQ1_M fills the same tile as IQ2_XS (int8 values, one scale per 16), so it takes IQ2_XS's configs+ if (type == GGML_TYPE_IQ1_M) {+ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, cc, prec_src1);+ if (config.type == GGML_TYPE_IQ2_XS) {+ config.type = GGML_TYPE_IQ1_M;+ }+ return config;+ }if (GGML_CUDA_CC_IS_AMD(cc)) {if (GGML_CUDA_CC_IS_GCN(cc)) {return ggml_cuda_mmq_get_config_gcn(type, J, fallback);@@ -273,6 +282,13 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty}static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ if (type == GGML_TYPE_IQ1_M) {+ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, prec_src1);+ if (config.type == GGML_TYPE_IQ2_XS) {+ config.type = GGML_TYPE_IQ1_M;+ }+ return config;+ }#ifdef GGML_USE_HIP#ifdef GCNreturn ggml_cuda_mmq_get_config_gcn(type, J, fallback);@@ -420,6 +436,7 @@ static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggmlcase GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ1_M: return MMQ_DP4A_TXS_Q8_0_16;case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;default: return tile_x_sizes{0, 0, 0};@@ -628,6 +645,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ1_M:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_XS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);case GGML_TYPE_IQ2_XXS:return ggml_cuda_mmq_util_funcs(VDR_IQ2_XXS_Q8_1_MMQ,@@ -798,6 +821,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ1_M:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);case GGML_TYPE_IQ2_XXS:return ggml_cuda_mmq_util_funcs(-1,@@ -1638,6 +1667,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);// -----------------------------------------extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);+extern DECL_MMQ_CASE(GGML_TYPE_IQ1_M);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_m.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_m.cunew file mode 100644index 00000000..f781ca77--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_m.cu@@ -0,0 +1,5 @@+// FreeToken: not generated, llama.cpp has no IQ1_M MMQ.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ1_M);diff --git a/tests/kernels/test_ggml.py b/tests/kernels/test_ggml.pyindex 8616a175..ae9ae309 100644--- a/tests/kernels/test_ggml.py+++ b/tests/kernels/test_ggml.py@@ -38,6 +38,29 @@ def _nvfp4_weights(shape, seed=765):return storage, reference+def _iq1_m_weights(shape, exact_sums, seed=1729):+ """Random IQ1_M blocks over every grid index, shift bit and 3-bit scale, padded for MMQ.++ Super-scales are random fp16 in [0.005, 0.02], every bit of which the loader decodes. With ``exact_sums`` they are+ m / 1024 (m in 5..20) instead: every product with ``_activation`` is then a multiple of 2^-19 and the sums are exact+ in fp32, so the kernel and the reference round a bf16 / fp16 result alike.+ """+ rng = np.random.default_rng(seed)+ *lead, rows, k = shape+ blocks = (*lead, rows, k // 256)+ qs = rng.integers(0, 256, (*blocks, 32), dtype=np.uint8)+ qh = rng.integers(0, 256, (*blocks, 16), dtype=np.uint8)+ d = rng.integers(5, 21, blocks) / 1024 if exact_sums else rng.uniform(0.005, 0.02, blocks)+ d = d.astype(np.float16).view(np.uint16)+ # the super-scale's four nibbles sit in the top nibble of the four uint16 scale words+ scales = rng.integers(0, 1 << 12, (*blocks, 4), dtype=np.uint16) | (((d[..., None] >> (4 * np.arange(4, dtype=np.uint16))) & 0xF) << 12)+ packed = np.concatenate([qs, qh, scales.astype("<u2").view(np.uint8)], axis=-1).reshape(*lead, rows, -1)+ reference = torch.from_numpy(quants.dequantize(packed, QType.IQ1_M))+ storage = torch.zeros(*lead, rows + 1, packed.shape[-1], dtype=torch.uint8, device="cuda")+ storage[..., :-1, :].copy_(torch.from_numpy(packed))+ return storage[..., :-1, :], reference++def _activation(shape, dtype, seed=19):rng = np.random.default_rng(seed)half = rng.integers(-127, 128, (*shape[:-1], shape[-1] // 32, 16))@@ -78,6 +101,22 @@ def test_q4_moe_routes_match_cpu(dtype, batch):torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)[email protected]("dtype", DTYPES)[email protected]("batch", [1, 17, 300])+def test_iq1_m_matrix_and_moe_routes_match_cpu(dtype, batch):+ # llama.cpp has no IQ1_M MMQ: above the vector kernel's batch this runs FreeToken's IQ1_M tile loader+ exact_sums = dtype != torch.float32+ w, ref = _iq1_m_weights((64, 512), exact_sums)+ x = _activation((batch, 512), dtype)+ actual = ggml.mul_mat(w, int(QType.IQ1_M), x)+ torch.testing.assert_close(actual.cpu(), F.linear(x.cpu().float(), ref).to(dtype), atol=2e-4, rtol=1e-3)+ w, ref = _iq1_m_weights((4, 64, 512), exact_sums)+ ids = torch.tensor([[i % 4, (i + 1) % 4] for i in range(batch)], dtype=torch.int32, device="cuda")+ actual = ggml.mul_mat_id(w, int(QType.IQ1_M), x, ids)+ expected = torch.einsum("tk, tjok -> tjo", x.cpu().float(), ref[ids.cpu().long()]).to(dtype)+ torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)++@pytest.mark.parametrize("dtype", DTYPES)def test_q4_shared_q8_matches_internal_quantization(dtype):w, _ = _q4_weights((64, 256))