Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(kernel): add an iq1_m mmq kernel for moe steps above 8 tokens (#835)

Chen Chengd2977fbc13c9
diff --git a/python/freetoken/kernel/csrc/ggml/VERSION b/python/freetoken/kernel/csrc/ggml/VERSION
index c4a9671d..114ad366 100644
--- a/python/freetoken/kernel/csrc/ggml/VERSION
+++ b/python/freetoken/kernel/csrc/ggml/VERSION
@@ -12,5 +12,6 @@ Local changes, kept in freetoken.patch (already applied):
- common.cuh: the native fp4 MMA (BLACKWELL_MMA_AVAILABLE, blackwell_mma_available) is off unless GGML_CUDA_FP4_MMA is defined; it needs an sm_120a/sm_120f build.
- mmvq.cu, common.cuh: the NVFP4 scale fusion (x_scale, gate_scale) reads strided scale vectors, e.g. columns of a per-expert scale bank.
- getrows.cu, dequantize.cuh: get_rows reads NVFP4 rows, e.g. an NVFP4 token_embd.
+- mmq.cu/.cuh, mmq-load-tiles.cuh, template-instances/mmq-instance-iq1_m.cu: MMQ runs IQ1_M, which llama.cpp has no MMQ kernel for. Its values times 8 are int8, so it loads into IQ2_XS's tile layout and takes IQ2_XS's configs.
To update: copy the same files from a newer llama.cpp, apply freetoken.patch, and update this file.
diff --git a/python/freetoken/kernel/csrc/ggml/freetoken.patch b/python/freetoken/kernel/csrc/ggml/freetoken.patch
index e968a078..e104a451 100644
--- a/python/freetoken/kernel/csrc/ggml/freetoken.patch
+++ b/python/freetoken/kernel/csrc/ggml/freetoken.patch
@@ -122,9 +122,98 @@
+
+void ggml_cuda_launch_mm_ids_tile_map(
+ const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream);
+--- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh
++++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh
+@@ -1095,6 +1095,76 @@
+ }
+ }
+
++// FreeToken: IQ1_M's delta changes every 8 values, so it cannot use IQ1_S's per-32 dm. 8 * (q + delta) is an integer
++// (8q - 7, or 8q - 9 with the shift bit), so the tile holds those int8 values and d * scale / 8 per 16, like IQ2_XS.
++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_m(
++ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
++ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
++ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
++
++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
++ int * x_qs = (int *) x_tile;
++ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
++#else
++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ1_M, I);
++ int * x_qs = (int *) x_tile;
++ float * x_df = (float *) (x_qs + txs.qs);
++#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
++
++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_M);
++ constexpr int nrows = warp_size / threads_per_row;
++ const int kqsx = threadIdx.x % threads_per_row;
++
++#pragma unroll
++ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
++ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
++
++ if (fallback) {
++ i = min(i, i_max);
++ }
++
++ const block_iq1_m * bxi = (const block_iq1_m *) x + kbx0 + i*stride;
++
++ const int qs_packed = get_int_b4(bxi->qs, kqsx);
++ const uint8_t * qs = (const uint8_t *) &qs_packed;
++
++ const int qh = bxi->qh[2*kqsx + 0] | (bxi->qh[2*kqsx + 1] << 8);
++
++ #pragma unroll
++ for (int l = 0; l < QR1_M/2; ++l) {
++ const int qhl = qh >> (4*l);
++ const int grid = iq1s_grid_gpu[qs[l] | ((qhl & 0x07) << 8)];
++ const int off = qhl & 0x08 ? 0x09090909 : 0x07070707;
++
++ const int grid0 = __vsub4(((grid >> 0) & 0x0F0F0F0F) << 3, off);
++ const int grid1 = __vsub4(((grid >> 4) & 0x0F0F0F0F) << 3, off);
++
++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
++ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;
++ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;
++#else
++ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;
++ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;
++#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
++ }
++
++ const uint16_t * sc = (const uint16_t *) bxi->scales;
++ iq1m_scale_t scale;
++ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
++ const float d = __half2float(scale.f16) / 8;
++ const int ls = sc[kqsx/2] >> (6*(kqsx%2));
++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
++ x_df[i*sram_stride + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);
++ x_df[i*sram_stride + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);
++#else
++ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);
++ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);
++#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
++ }
++}
++
+ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
--- a/ggml/src/ggml-cuda/mmq.cu
+++ b/ggml/src/ggml-cuda/mmq.cu
-@@ -132,9 +132,11 @@
+@@ -48,6 +48,9 @@
+ case GGML_TYPE_IQ1_S:
+ mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);
+ break;
++ case GGML_TYPE_IQ1_M:
++ mul_mat_q_case<GGML_TYPE_IQ1_M, GGML_PREC_Q8>(ctx, args, stream);
++ break;
+ case GGML_TYPE_IQ2_XXS:
+ mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+@@ -132,9 +135,11 @@
}
void ggml_cuda_mul_mat_q(
@@ -139,7 +228,7 @@
GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
GGML_TENSOR_BINARY_OP_LOCALS;
-@@ -156,7 +158,9 @@
+@@ -156,7 +161,9 @@
const char * src0_d = (const char *) src0->data;
const float * src1_d = (const float *) src1->data;
@@ -150,7 +239,7 @@
// If src0 is a temporary compute buffer, clear any potential padding.
if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
-@@ -183,6 +187,7 @@
+@@ -183,6 +190,7 @@
const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);
const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;
@@ -158,7 +247,7 @@
const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);
const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ;
-@@ -192,7 +197,8 @@
+@@ -192,7 +200,8 @@
int64_t ncols_opt = ne11;
if (ids) {
const int64_t n_expert_used = ids->ne[0];
@@ -168,7 +257,7 @@
// Each expert only sees ne12*n_expert_used/ne02 tokens on average.
// On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.
-@@ -249,6 +255,12 @@
+@@ -249,6 +258,12 @@
quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,
ne11, ne12, ne13, stream);
@@ -181,7 +270,7 @@
} else {
quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
ne11, ne12, ne13, stream);
-@@ -299,6 +311,17 @@
+@@ -299,6 +314,17 @@
CUDA_CHECK(cudaGetLastError());
}
@@ -199,7 +288,7 @@
const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;
ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());
-@@ -325,9 +348,21 @@
+@@ -325,9 +351,21 @@
quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,
ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
}
@@ -221,7 +310,7 @@
} else {
quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
-@@ -347,7 +382,7 @@
+@@ -347,7 +385,7 @@
ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,
ne02, ne02, s02, s12, s2,
ne03, ne13, s03, s13, s3,
@@ -230,6 +319,14 @@
ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
}
+@@ -375,6 +413,7 @@
+ case GGML_TYPE_Q6_K:
+ // -------------------------------------------------
+ case GGML_TYPE_IQ1_S:
++ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
--- a/ggml/src/ggml-cuda/mmq.cuh
+++ b/ggml/src/ggml-cuda/mmq.cuh
@@ -10,6 +10,17 @@
@@ -250,7 +347,52 @@
typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);
typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);
typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,
-@@ -429,8 +440,8 @@
+@@ -88,6 +99,7 @@
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ3_S:
++ case GGML_TYPE_IQ1_M:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_IQ1_S:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+@@ -227,6 +239,14 @@
+ #undef CASE
+
+ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
++ // FreeToken: IQ1_M fills the same tile as IQ2_XS (int8 values, one scale per 16), so it takes IQ2_XS's configs
++ if (type == GGML_TYPE_IQ1_M) {
++ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, cc, prec_src1);
++ if (config.type == GGML_TYPE_IQ2_XS) {
++ config.type = GGML_TYPE_IQ1_M;
++ }
++ return config;
++ }
+ if (GGML_CUDA_CC_IS_AMD(cc)) {
+ if (GGML_CUDA_CC_IS_GCN(cc)) {
+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
+@@ -262,6 +282,13 @@
+ }
+
+ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
++ if (type == GGML_TYPE_IQ1_M) {
++ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, prec_src1);
++ if (config.type == GGML_TYPE_IQ2_XS) {
++ config.type = GGML_TYPE_IQ1_M;
++ }
++ return config;
++ }
+ #ifdef GGML_USE_HIP
+ #ifdef GCN
+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
+@@ -409,6 +436,7 @@
+ case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;
++ case GGML_TYPE_IQ1_M: return MMQ_DP4A_TXS_Q8_0_16;
+ case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;
+ default: return tile_x_sizes{0, 0, 0};
+@@ -429,8 +457,8 @@
#include "mmq-load-tiles.cuh"
#include "mmq-vec-dot.cuh"
@@ -261,7 +403,7 @@
const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
-@@ -456,21 +467,21 @@
+@@ -456,21 +484,21 @@
if constexpr (type == GGML_TYPE_NVFP4) {
if (y_scale_used) {
@@ -288,7 +430,7 @@
const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
-@@ -506,12 +517,12 @@
+@@ -506,12 +534,12 @@
if constexpr (type == GGML_TYPE_NVFP4) {
if (y_scale_used) {
@@ -304,7 +446,33 @@
GGML_UNUSED(y_scale_used);
}
}
-@@ -869,10 +880,10 @@
+@@ -617,6 +645,12 @@
+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
++ case GGML_TYPE_IQ1_M:
++ return ggml_cuda_mmq_util_funcs(
++ VDR_IQ2_XS_Q8_1_MMQ,
++ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,
++ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
++ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_XXS:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ2_XXS_Q8_1_MMQ,
+@@ -787,6 +821,12 @@
+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
++ case GGML_TYPE_IQ1_M:
++ return ggml_cuda_mmq_util_funcs(
++ -1,
++ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,
++ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
++ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_XXS:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+@@ -869,10 +909,10 @@
// ---------------------------------------------------------------------------------------------
@@ -317,7 +485,7 @@
const float * __restrict__ y_scale,
const int stride_row_x, const int ncols_y, const int stride_col_dst,
const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {
-@@ -942,19 +953,32 @@
+@@ -942,19 +982,32 @@
if (fixup) {
write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);
@@ -353,7 +521,7 @@
const float * __restrict__ y_scale,
const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,
const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
-@@ -993,9 +1017,10 @@
+@@ -993,9 +1046,10 @@
if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
const int wt = tmp2.x;
@@ -366,7 +534,7 @@
// Defaults for regular matrix multiplication:
int col_low = 0;
-@@ -1076,14 +1101,15 @@
+@@ -1076,14 +1130,15 @@
while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {
int tmp = fastdiv(kbc, blocks_per_ne00);
uint2 tmp2 = fast_div_modulo(tmp, ntx);
@@ -384,7 +552,7 @@
// Defaults for regular matrix multiplication:
int col_low = 0;
-@@ -1165,14 +1191,15 @@
+@@ -1165,14 +1220,15 @@
int tmp = fastdiv(kbc, blocks_per_ne00);
uint2 tmp2 = fast_div_modulo(tmp, ntx);
@@ -402,7 +570,7 @@
// Defaults for regular matrix multiplication:
int col_low = 0;
-@@ -1375,14 +1402,26 @@
+@@ -1375,14 +1431,26 @@
}
}
@@ -430,7 +598,7 @@
};
static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {
-@@ -1406,12 +1445,11 @@
+@@ -1406,12 +1474,11 @@
const dim3 block_dims(warp_size, nwarps, 1);
@@ -447,7 +615,7 @@
const dim3 block_nums_xy_tiling(nty, ntx, ntzw);
GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);
-@@ -1421,18 +1459,71 @@
+@@ -1421,18 +1488,71 @@
const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);
const uint3 ntx_fd = init_fastdiv_values(ntx);
@@ -523,7 +691,7 @@
return;
}
-@@ -1441,7 +1532,8 @@
+@@ -1441,7 +1561,8 @@
const int ntiles_dst = ntx * nty * ntzw;
const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
@@ -533,7 +701,7 @@
GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
-@@ -1453,25 +1545,7 @@
+@@ -1453,25 +1574,7 @@
tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);
}
@@ -560,7 +728,15 @@
}
template <ggml_type type, bool fallback, ggml_prec prec_src1>
-@@ -1579,7 +1653,9 @@
+@@ -1564,6 +1667,7 @@
+ extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
+ // -----------------------------------------
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
++extern DECL_MMQ_CASE(GGML_TYPE_IQ1_M);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
+@@ -1579,7 +1683,9 @@
// -------------------------------------------------------------------------------------------------------------------------
@@ -1087,3 +1263,11 @@
const int32_t * ids_src1_inv,
void * vy,
ggml_type type_src0,
+--- /dev/null
++++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_m.cu
+@@ -0,0 +1,5 @@
++// FreeToken: not generated, llama.cpp has no IQ1_M MMQ.
++
++#include "../mmq.cuh"
++
++DECL_MMQ_CASE(GGML_TYPE_IQ1_M);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh
index 017b93b5..4dd92170 100644
--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh
@@ -1095,6 +1095,76 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
}
}
+// FreeToken: IQ1_M's delta changes every 8 values, so it cannot use IQ1_S's per-32 dm. 8 * (q + delta) is an integer
+// (8q - 7, or 8q - 9 with the shift bit), so the tile holds those int8 values and d * scale / 8 per 16, like IQ2_XS.
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_m(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ1_M, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_M);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq1_m * bxi = (const block_iq1_m *) x + kbx0 + i*stride;
+
+ const int qs_packed = get_int_b4(bxi->qs, kqsx);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[2*kqsx + 0] | (bxi->qh[2*kqsx + 1] << 8);
+
+ #pragma unroll
+ for (int l = 0; l < QR1_M/2; ++l) {
+ const int qhl = qh >> (4*l);
+ const int grid = iq1s_grid_gpu[qs[l] | ((qhl & 0x07) << 8)];
+ const int off = qhl & 0x08 ? 0x09090909 : 0x07070707;
+
+ const int grid0 = __vsub4(((grid >> 0) & 0x0F0F0F0F) << 3, off);
+ const int grid1 = __vsub4(((grid >> 4) & 0x0F0F0F0F) << 3, off);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const uint16_t * sc = (const uint16_t *) bxi->scales;
+ iq1m_scale_t scale;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
+ const float d = __half2float(scale.f16) / 8;
+ const int ls = sc[kqsx/2] >> (6*(kqsx%2));
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);
+ x_df[i*sram_stride + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);
+#else
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(
const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu
index f754837d..b1c424de 100644
--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu
@@ -48,6 +48,9 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
case GGML_TYPE_IQ1_S:
mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);
break;
+ case GGML_TYPE_IQ1_M:
+ mul_mat_q_case<GGML_TYPE_IQ1_M, GGML_PREC_Q8>(ctx, args, stream);
+ break;
case GGML_TYPE_IQ2_XXS:
mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream);
break;
@@ -410,6 +413,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
case GGML_TYPE_Q6_K:
// -------------------------------------------------
case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
case GGML_TYPE_IQ2_S:
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh
index 596a54ea..154a628a 100644
--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh
@@ -99,6 +99,7 @@ static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {
case GGML_TYPE_IQ2_S:
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ1_M:
return MMQ_Q8_1_DS_LAYOUT_D4;
case GGML_TYPE_IQ1_S:
return MMQ_Q8_1_DS_LAYOUT_DS4;
@@ -238,6 +239,14 @@ struct ggml_cuda_mmq_config {
#undef CASE
static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ // FreeToken: IQ1_M fills the same tile as IQ2_XS (int8 values, one scale per 16), so it takes IQ2_XS's configs
+ if (type == GGML_TYPE_IQ1_M) {
+ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, cc, prec_src1);
+ if (config.type == GGML_TYPE_IQ2_XS) {
+ config.type = GGML_TYPE_IQ1_M;
+ }
+ return config;
+ }
if (GGML_CUDA_CC_IS_AMD(cc)) {
if (GGML_CUDA_CC_IS_GCN(cc)) {
return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
@@ -273,6 +282,13 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty
}
static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ if (type == GGML_TYPE_IQ1_M) {
+ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, prec_src1);
+ if (config.type == GGML_TYPE_IQ2_XS) {
+ config.type = GGML_TYPE_IQ1_M;
+ }
+ return config;
+ }
#ifdef GGML_USE_HIP
#ifdef GCN
return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
@@ -420,6 +436,7 @@ static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml
case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;
case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;
case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ1_M: return MMQ_DP4A_TXS_Q8_0_16;
case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;
case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;
default: return tile_x_sizes{0, 0, 0};
@@ -628,6 +645,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,
ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ1_M:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ2_XS_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
case GGML_TYPE_IQ2_XXS:
return ggml_cuda_mmq_util_funcs(
VDR_IQ2_XXS_Q8_1_MMQ,
@@ -798,6 +821,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ1_M:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
case GGML_TYPE_IQ2_XXS:
return ggml_cuda_mmq_util_funcs(
-1,
@@ -1638,6 +1667,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
// -----------------------------------------
extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ1_M);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_m.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_m.cu
new file mode 100644
index 00000000..f781ca77
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_m.cu
@@ -0,0 +1,5 @@
+// FreeToken: not generated, llama.cpp has no IQ1_M MMQ.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ1_M);
diff --git a/tests/kernels/test_ggml.py b/tests/kernels/test_ggml.py
index 8616a175..ae9ae309 100644
--- a/tests/kernels/test_ggml.py
+++ b/tests/kernels/test_ggml.py
@@ -38,6 +38,29 @@ def _nvfp4_weights(shape, seed=765):
return storage, reference
+def _iq1_m_weights(shape, exact_sums, seed=1729):
+ """Random IQ1_M blocks over every grid index, shift bit and 3-bit scale, padded for MMQ.
+
+ Super-scales are random fp16 in [0.005, 0.02], every bit of which the loader decodes. With ``exact_sums`` they are
+ m / 1024 (m in 5..20) instead: every product with ``_activation`` is then a multiple of 2^-19 and the sums are exact
+ in fp32, so the kernel and the reference round a bf16 / fp16 result alike.
+ """
+ rng = np.random.default_rng(seed)
+ *lead, rows, k = shape
+ blocks = (*lead, rows, k // 256)
+ qs = rng.integers(0, 256, (*blocks, 32), dtype=np.uint8)
+ qh = rng.integers(0, 256, (*blocks, 16), dtype=np.uint8)
+ d = rng.integers(5, 21, blocks) / 1024 if exact_sums else rng.uniform(0.005, 0.02, blocks)
+ d = d.astype(np.float16).view(np.uint16)
+ # the super-scale's four nibbles sit in the top nibble of the four uint16 scale words
+ scales = rng.integers(0, 1 << 12, (*blocks, 4), dtype=np.uint16) | (((d[..., None] >> (4 * np.arange(4, dtype=np.uint16))) & 0xF) << 12)
+ packed = np.concatenate([qs, qh, scales.astype("<u2").view(np.uint8)], axis=-1).reshape(*lead, rows, -1)
+ reference = torch.from_numpy(quants.dequantize(packed, QType.IQ1_M))
+ storage = torch.zeros(*lead, rows + 1, packed.shape[-1], dtype=torch.uint8, device="cuda")
+ storage[..., :-1, :].copy_(torch.from_numpy(packed))
+ return storage[..., :-1, :], reference
+
+
def _activation(shape, dtype, seed=19):
rng = np.random.default_rng(seed)
half = rng.integers(-127, 128, (*shape[:-1], shape[-1] // 32, 16))
@@ -78,6 +101,22 @@ def test_q4_moe_routes_match_cpu(dtype, batch):
torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)
[email protected]("dtype", DTYPES)
[email protected]("batch", [1, 17, 300])
+def test_iq1_m_matrix_and_moe_routes_match_cpu(dtype, batch):
+ # llama.cpp has no IQ1_M MMQ: above the vector kernel's batch this runs FreeToken's IQ1_M tile loader
+ exact_sums = dtype != torch.float32
+ w, ref = _iq1_m_weights((64, 512), exact_sums)
+ x = _activation((batch, 512), dtype)
+ actual = ggml.mul_mat(w, int(QType.IQ1_M), x)
+ torch.testing.assert_close(actual.cpu(), F.linear(x.cpu().float(), ref).to(dtype), atol=2e-4, rtol=1e-3)
+ w, ref = _iq1_m_weights((4, 64, 512), exact_sums)
+ ids = torch.tensor([[i % 4, (i + 1) % 4] for i in range(batch)], dtype=torch.int32, device="cuda")
+ actual = ggml.mul_mat_id(w, int(QType.IQ1_M), x, ids)
+ expected = torch.einsum("tk, tjok -> tjo", x.cpu().float(), ref[ids.cpu().long()]).to(dtype)
+ torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)
+
+
@pytest.mark.parametrize("dtype", DTYPES)
def test_q4_shared_q8_matches_internal_quantization(dtype):
w, _ = _q4_weights((64, 256))