FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
--- a/ggml/src/ggml-cuda/common.cuh
+++ b/ggml/src/ggml-cuda/common.cuh
@@ -293,7 +293,8 @@
#define AMPERE_MMA_AVAILABLE
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
-#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN
+// FreeToken: the native fp4 MMA needs an sm_120a/sm_120f build, so it is off unless GGML_CUDA_FP4_MMA is defined; FP4 weights then take Q8_1 activations
+#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN && defined(GGML_CUDA_FP4_MMA)
# define BLACKWELL_MMA_AVAILABLE
#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL
@@ -374,8 +375,13 @@
}
static bool blackwell_mma_available(const int cc) {
+#ifdef GGML_CUDA_FP4_MMA
return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL &&
ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;
+#else
+ GGML_UNUSED(cc);
+ return false;
+#endif // GGML_CUDA_FP4_MMA
}
// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.
@@ -1582,6 +1588,9 @@
const void * gate_bias = nullptr;
const void * x_scale = nullptr;
const void * gate_scale = nullptr;
+ // FreeToken: element strides of x_scale / gate_scale, so they can be columns of a per-expert scale bank
+ uint32_t x_scale_stride = 1;
+ uint32_t gate_scale_stride = 1;
ggml_glu_op glu_op;
float glu_limit = 0.0f;
const void * shared_up = nullptr;
--- a/ggml/src/ggml-cuda/dequantize.cuh
+++ b/ggml/src/ggml-cuda/dequantize.cuh
@@ -450,3 +450,15 @@
y[j+16] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q4[j] >> 4]*0.5f);
}
}
+
+// FreeToken: for k_get_rows with qk = QK_NVFP4_SUB, so ib indexes a 16-element sub-block and the nibble pair lands at iqs and iqs + 8
+static __device__ __forceinline__ void dequantize_nvfp4(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_nvfp4 * x = (const block_nvfp4 *) vx + ib/(QK_NVFP4/QK_NVFP4_SUB);
+ const int sub = ib % (QK_NVFP4/QK_NVFP4_SUB);
+
+ const float d = ggml_cuda_ue4m3_to_fp32(x->d[sub]);
+ const int vui = x->qs[sub*(QK_NVFP4_SUB/2) + iqs];
+
+ v.x = d * kvalues_mxfp4[vui & 0xF];
+ v.y = d * kvalues_mxfp4[vui >> 4];
+}
--- a/ggml/src/ggml-cuda/getrows.cu
+++ b/ggml/src/ggml-cuda/getrows.cu
@@ -404,6 +404,10 @@
get_rows_cuda_kq<32, dst_t, dequantize_mxfp4<dst_t>>(src0_d, src1_d, dst_d,
ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
break;
+ case GGML_TYPE_NVFP4:
+ get_rows_cuda_q<QK_NVFP4_SUB, QR_NVFP4, dequantize_nvfp4>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
default:
GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0_type));
}
--- a/ggml/src/ggml-cuda/mmid.cu
+++ b/ggml/src/ggml-cuda/mmid.cu
@@ -177,3 +177,45 @@
break;
}
}
+
+// Lists the (expert, column tile) pairs that hold columns for a flat MoE grid, in expert order.
+// The other ntiles grid columns get tile jt_none of expert 0, which starts past its columns, so mul_mat_q skips them.
+template <int nthreads>
+__launch_bounds__(nthreads, 1)
+static __global__ void mm_ids_tile_map(
+ const int32_t * __restrict__ expert_bounds, int2 * __restrict__ tile_map, const int n_experts, const int J, const int ntiles, const int jt_none) {
+ __shared__ int scan[nthreads];
+
+ int start = 0; // First tile of the current chunk of experts.
+ for (int e0 = 0; e0 < n_experts; e0 += nthreads) {
+ const int e = e0 + threadIdx.x;
+ const int n = e < n_experts ? (expert_bounds[e + 1] - expert_bounds[e] + J - 1) / J : 0;
+
+ scan[threadIdx.x] = n;
+ __syncthreads();
+#pragma unroll
+ for (int offset = 1; offset < nthreads; offset *= 2) {
+ const int add = threadIdx.x >= offset ? scan[threadIdx.x - offset] : 0;
+ __syncthreads();
+ scan[threadIdx.x] += add;
+ __syncthreads();
+ }
+
+ const int first = start + scan[threadIdx.x] - n;
+ for (int jt = 0; jt < n; ++jt) {
+ tile_map[first + jt] = make_int2(e, jt);
+ }
+ start += scan[nthreads - 1];
+ __syncthreads();
+ }
+
+ for (int t = start + threadIdx.x; t < ntiles; t += nthreads) {
+ tile_map[t] = make_int2(0, jt_none);
+ }
+}
+
+void ggml_cuda_launch_mm_ids_tile_map(
+ const int32_t * expert_bounds, int2 * tile_map, const int n_experts, const int J, const int ntiles, const int jt_none, cudaStream_t stream) {
+ constexpr int nthreads = 512;
+ mm_ids_tile_map<nthreads><<<1, nthreads, 0, stream>>>(expert_bounds, tile_map, n_experts, J, ntiles, jt_none);
+}
--- a/ggml/src/ggml-cuda/mmid.cuh
+++ b/ggml/src/ggml-cuda/mmid.cuh
@@ -3,3 +3,6 @@
void ggml_cuda_launch_mm_ids_helper(
const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds,
int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream);
+
+void ggml_cuda_launch_mm_ids_tile_map(
+ const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream);
--- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh
+++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh
@@ -1095,6 +1095,76 @@
}
}
+// FreeToken: IQ1_M's delta changes every 8 values, so it cannot use IQ1_S's per-32 dm. 8 * (q + delta) is an integer
+// (8q - 7, or 8q - 9 with the shift bit), so the tile holds those int8 values and d * scale / 8 per 16, like IQ2_XS.
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_m(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ1_M, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_M);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq1_m * bxi = (const block_iq1_m *) x + kbx0 + i*stride;
+
+ const int qs_packed = get_int_b4(bxi->qs, kqsx);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[2*kqsx + 0] | (bxi->qh[2*kqsx + 1] << 8);
+
+ #pragma unroll
+ for (int l = 0; l < QR1_M/2; ++l) {
+ const int qhl = qh >> (4*l);
+ const int grid = iq1s_grid_gpu[qs[l] | ((qhl & 0x07) << 8)];
+ const int off = qhl & 0x08 ? 0x09090909 : 0x07070707;
+
+ const int grid0 = __vsub4(((grid >> 0) & 0x0F0F0F0F) << 3, off);
+ const int grid1 = __vsub4(((grid >> 4) & 0x0F0F0F0F) << 3, off);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const uint16_t * sc = (const uint16_t *) bxi->scales;
+ iq1m_scale_t scale;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
+ const float d = __half2float(scale.f16) / 8;
+ const int ls = sc[kqsx/2] >> (6*(kqsx%2));
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);
+ x_df[i*sram_stride + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);
+#else
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1);
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(
const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
--- a/ggml/src/ggml-cuda/mmq.cu
+++ b/ggml/src/ggml-cuda/mmq.cu
@@ -48,6 +48,9 @@
case GGML_TYPE_IQ1_S:
mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);
break;
+ case GGML_TYPE_IQ1_M:
+ mul_mat_q_case<GGML_TYPE_IQ1_M, GGML_PREC_Q8>(ctx, args, stream);
+ break;
case GGML_TYPE_IQ2_XXS:
mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream);
break;
@@ -132,9 +135,11 @@
}
void ggml_cuda_mul_mat_q(
- ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
- GGML_ASSERT( src1->type == GGML_TYPE_F32);
- GGML_ASSERT( dst->type == GGML_TYPE_F32);
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
+ int64_t ncols_max) {
+ // FreeToken: bf16 and fp16 activations are quantized directly, except for the native fp4 paths; dst is fp32, bf16 or fp16
+ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);
GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
GGML_TENSOR_BINARY_OP_LOCALS;
@@ -156,7 +161,9 @@
const char * src0_d = (const char *) src0->data;
const float * src1_d = (const float *) src1->data;
- float * dst_d = (float *) dst->data;
+ const mmq_dst dst_d = {dst->data, dst->type};
+ const nv_bfloat16 * src1_bf16 = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1->data : nullptr;
+ const half * src1_f16 = src1->type == GGML_TYPE_F16 ? (const half *) src1->data : nullptr;
// If src0 is a temporary compute buffer, clear any potential padding.
if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
@@ -183,6 +190,7 @@
const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);
const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;
+ GGML_ASSERT(!use_native_fp4 || src1->type == GGML_TYPE_F32);
const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);
const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ;
@@ -192,7 +200,8 @@
int64_t ncols_opt = ne11;
if (ids) {
const int64_t n_expert_used = ids->ne[0];
- ncols_opt = ne12;
+ // a flat grid runs only the tiles that hold tokens, so it sizes them for the average expert
+ ncols_opt = ncols_max > 0 ? ncols_max : (ne12*n_expert_used + ne02 - 1) / ne02;
// Each expert only sees ne12*n_expert_used/ne02 tokens on average.
// On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.
@@ -249,6 +258,12 @@
quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,
ne11, ne12, ne13, stream);
+ } else if (src1_bf16) {
+ quantize_mmq_q8_1_cuda(src1_bf16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
+ } else if (src1_f16) {
+ quantize_mmq_q8_1_cuda(src1_f16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
} else {
quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
ne11, ne12, ne13, stream);
@@ -299,6 +314,17 @@
CUDA_CHECK(cudaGetLastError());
}
+ // Every expert's column tiles fit in ntx_flat grid columns, so a flat grid needs no host sync to size it.
+ ggml_cuda_pool_alloc<int2> tile_map(ctx.pool());
+ int ntx_flat = 0;
+ if (ncols_max <= 0) {
+ const int jt_none = (ne_get_rows + J_best - 1) / J_best;
+ ntx_flat = jt_none + std::min<int64_t>(ne02, ne_get_rows);
+ tile_map.alloc(ntx_flat);
+ ggml_cuda_launch_mm_ids_tile_map(expert_bounds.get(), tile_map.get(), ne02, J_best, ntx_flat, jt_none, stream);
+ CUDA_CHECK(cudaGetLastError());
+ }
+
const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;
ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());
@@ -325,9 +351,21 @@
quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,
ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
}
+ } else if (dedup_bcast && src1_bf16) {
+ quantize_scatter_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
+ } else if (dedup_bcast && src1_f16) {
+ quantize_scatter_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
} else if (dedup_bcast) {
quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
/*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
+ } else if (src1_bf16) {
+ quantize_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+ } else if (src1_f16) {
+ quantize_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
} else {
quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
@@ -347,7 +385,7 @@
ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,
ne02, ne02, s02, s12, s2,
ne03, ne13, s03, s13, s3,
- ne12, J_best};
+ ncols_max > 0 ? ncols_max : ne12, J_best, tile_map.ptr, ntx_flat};
ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
}
@@ -375,6 +413,7 @@
case GGML_TYPE_Q6_K:
// -------------------------------------------------
case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
case GGML_TYPE_IQ2_S:
--- a/ggml/src/ggml-cuda/mmq.cuh
+++ b/ggml/src/ggml-cuda/mmq.cuh
@@ -10,6 +10,17 @@
#define MMQ_ITER_K_FP4 512
#define MMQ_NWARPS 8
+// FreeToken: MMQ writes fp32, bf16 or fp16 results (dst_t); the sums stay fp32 until the store
+template <typename T> static __device__ __forceinline__ T mmq_cast(float v);
+template <> __device__ __forceinline__ float mmq_cast<float>(float v) { return v; }
+template <> __device__ __forceinline__ nv_bfloat16 mmq_cast<nv_bfloat16>(float v) { return __float2bfloat16(v); }
+template <> __device__ __forceinline__ half mmq_cast<half>(float v) { return __float2half(v); }
+
+struct mmq_dst {
+ void * p;
+ ggml_type type;
+};
+
typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);
typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);
typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,
@@ -88,6 +99,7 @@
case GGML_TYPE_IQ2_S:
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ1_M:
return MMQ_Q8_1_DS_LAYOUT_D4;
case GGML_TYPE_IQ1_S:
return MMQ_Q8_1_DS_LAYOUT_DS4;
@@ -227,6 +239,14 @@
#undef CASE
static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ // FreeToken: IQ1_M fills the same tile as IQ2_XS (int8 values, one scale per 16), so it takes IQ2_XS's configs
+ if (type == GGML_TYPE_IQ1_M) {
+ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, cc, prec_src1);
+ if (config.type == GGML_TYPE_IQ2_XS) {
+ config.type = GGML_TYPE_IQ1_M;
+ }
+ return config;
+ }
if (GGML_CUDA_CC_IS_AMD(cc)) {
if (GGML_CUDA_CC_IS_GCN(cc)) {
return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
@@ -262,6 +282,13 @@
}
static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ if (type == GGML_TYPE_IQ1_M) {
+ ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, prec_src1);
+ if (config.type == GGML_TYPE_IQ2_XS) {
+ config.type = GGML_TYPE_IQ1_M;
+ }
+ return config;
+ }
#ifdef GGML_USE_HIP
#ifdef GCN
return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
@@ -409,6 +436,7 @@
case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;
case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;
case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ1_M: return MMQ_DP4A_TXS_Q8_0_16;
case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;
case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;
default: return tile_x_sizes{0, 0, 0};
@@ -429,8 +457,8 @@
#include "mmq-load-tiles.cuh"
#include "mmq-vec-dot.cuh"
-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(
- const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(
+ const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, dst_t * __restrict__ dst,
const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
@@ -456,21 +484,21 @@
if constexpr (type == GGML_TYPE_NVFP4) {
if (y_scale_used) {
- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
} else {
- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
}
} else {
- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
GGML_UNUSED(y_scale_used);
}
}
}
}
-template<ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+template<ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float>
static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
- const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,
+ const float * __restrict__ sum, const int * __restrict__ ids_dst, dst_t * __restrict__ dst,
const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
@@ -506,12 +534,12 @@
if constexpr (type == GGML_TYPE_NVFP4) {
if (y_scale_used) {
- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l];
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]);
} else {
- dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);
}
} else {
- dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);
GGML_UNUSED(y_scale_used);
}
}
@@ -617,6 +645,12 @@
ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,
ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ1_M:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ2_XS_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
case GGML_TYPE_IQ2_XXS:
return ggml_cuda_mmq_util_funcs(
VDR_IQ2_XXS_Q8_1_MMQ,
@@ -787,6 +821,12 @@
ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ1_M:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq1_m<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
case GGML_TYPE_IQ2_XXS:
return ggml_cuda_mmq_util_funcs(
-1,
@@ -869,10 +909,10 @@
// ---------------------------------------------------------------------------------------------
-template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1>
+template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1, typename dst_t>
static __device__ __forceinline__ void mul_mat_q_process_tile(
const char * __restrict__ x, const int offset_x, const int * __restrict__ y,
- const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const int * __restrict__ ids_dst, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,
const float * __restrict__ y_scale,
const int stride_row_x, const int ncols_y, const int stride_col_dst,
const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {
@@ -942,19 +982,32 @@
if (fixup) {
write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);
- } else {
+ } else if constexpr (std::is_same_v<dst_t, float>) {
write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
+ } else if constexpr (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
+ } else {
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
}
}
+// A flat MoE grid runs only the (expert, column tile) pairs listed in tile_map, see ggml_cuda_launch_mm_ids_tile_map.
+static __device__ __forceinline__ void mmq_flat_tile(const int2 * __restrict__ tile_map, int & zt, int & jt) {
+ if (tile_map) {
+ const int2 t = tile_map[jt];
+ zt = t.x;
+ jt = t.y;
+ }
+}
+
// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t>
__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1))
static __global__ void mul_mat_q(
const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
- const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const int32_t * __restrict__ expert_bounds, const int2 * __restrict__ tile_map, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,
const float * __restrict__ y_scale,
const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,
const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
@@ -993,9 +1046,10 @@
if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
const int wt = tmp2.x;
- const int zt = tmp2.y;
- const int jt = blockIdx.y;
+ int zt = tmp2.y;
+ int jt = blockIdx.y;
const int it = blockIdx.x;
+ mmq_flat_tile(tile_map, zt, jt);
// Defaults for regular matrix multiplication:
int col_low = 0;
@@ -1076,14 +1130,15 @@
while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {
int tmp = fastdiv(kbc, blocks_per_ne00);
uint2 tmp2 = fast_div_modulo(tmp, ntx);
- const int jt = tmp2.y;
+ int jt = tmp2.y;
tmp = tmp2.x;
tmp2 = fast_div_modulo(tmp, nchannels_y);
- const int zt = tmp2.y;
+ int zt = tmp2.y;
tmp = tmp2.x;
tmp2 = fast_div_modulo(tmp, nsamples_y);
const int wt = tmp2.y;
const int it = tmp2.x;
+ mmq_flat_tile(tile_map, zt, jt);
// Defaults for regular matrix multiplication:
int col_low = 0;
@@ -1165,14 +1220,15 @@
int tmp = fastdiv(kbc, blocks_per_ne00);
uint2 tmp2 = fast_div_modulo(tmp, ntx);
- const int jt = tmp2.y;
+ int jt = tmp2.y;
tmp = tmp2.x;
tmp2 = fast_div_modulo(tmp, nchannels_y);
- const int zt = tmp2.y;
+ int zt = tmp2.y;
tmp = tmp2.x;
tmp2 = fast_div_modulo(tmp, nsamples_y);
const int wt = tmp2.y;
const int it = tmp2.x;
+ mmq_flat_tile(tile_map, zt, jt);
// Defaults for regular matrix multiplication:
int col_low = 0;
@@ -1375,14 +1431,26 @@
}
}
+// FreeToken: copies an fp32 result with dst's layout into dst
+template <typename dst_t>
+static __global__ void mmq_store_dst(const float * __restrict__ src, dst_t * __restrict__ dst, const int nrows, const int stride) {
+ const int64_t col = blockIdx.x;
+ const int row = blockIdx.y*blockDim.x + threadIdx.x;
+ if (row < nrows) {
+ dst[col*stride + row] = mmq_cast<dst_t>(src[col*stride + row]);
+ }
+}
+
struct mmq_args {
- const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst;
+ const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; mmq_dst dst;
const float * y_scale;
int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;
int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;
int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;
int64_t ncols_max;
int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance.
+ const int2 * tile_map = nullptr; // Flat MoE grid: the (expert, column tile) of each of the ntx_flat grid columns.
+ int ntx_flat = 0;
};
static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {
@@ -1406,12 +1474,11 @@
const dim3 block_dims(warp_size, nwarps, 1);
- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1>), nbytes_shared);
- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1>), nbytes_shared);
-
+ // a flat MoE grid puts all experts into one channel of ntx_flat column tiles
+ const int64_t nchannels_grid = args.tile_map ? 1 : args.nchannels_y;
const int nty = (args.nrows_x + config.I - 1) / config.I;
- const int ntx = (args.ncols_max + config.J - 1) / config.J;
- const int ntzw = args.nchannels_y * args.nsamples_y;
+ const int ntx = args.tile_map ? args.ntx_flat : (args.ncols_max + config.J - 1) / config.J;
+ const int ntzw = nchannels_grid * args.nsamples_y;
const dim3 block_nums_xy_tiling(nty, ntx, ntzw);
GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);
@@ -1421,18 +1488,71 @@
const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);
const uint3 ntx_fd = init_fastdiv_values(ntx);
- const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y);
+ const uint3 nchannels_y_fd = init_fastdiv_values(nchannels_grid);
const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y);
const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio);
const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);
- if (!config.stream_k) {
- mul_mat_q<type, J, fallback, prec_src1><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale,
+ const auto run = [&](auto * dst, const dim3 & block_nums, float * tmp_fixup) {
+ using dst_t = std::remove_pointer_t<decltype(dst)>;
+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1, dst_t>), nbytes_shared);
+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1, dst_t>), nbytes_shared);
+ mul_mat_q<type, J, fallback, prec_src1, dst_t><<<block_nums, block_dims, nbytes_shared, stream>>>
+ (args.x, args.y, args.ids_dst, args.expert_bounds, args.tile_map, dst, tmp_fixup, args.y_scale,
blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
ntx_fd);
+ };
+
+ // FreeToken: the fixup kernel adds partial sums into an fp32 dst, so a bf16 dst that needs it is computed in fp32 first.
+ // An fp16 dst always is, unless GGML_MMQ_F16_DST builds the fp16 kernels.
+ const auto launch = [&](const dim3 & block_nums, const bool fixup, float * tmp_fixup) {
+ bool via_f32 = fixup && args.dst.type != GGML_TYPE_F32;
+#ifndef GGML_MMQ_F16_DST
+ via_f32 = via_f32 || args.dst.type == GGML_TYPE_F16;
+#endif // GGML_MMQ_F16_DST
+ ggml_cuda_pool_alloc<float> dst_f32(ctx.pool(id));
+ float * dst_float = args.dst.type == GGML_TYPE_F32 ? (float *) args.dst.p : nullptr;
+ if (via_f32) {
+ GGML_ASSERT(args.ids_dst || (args.nchannels_y == 1 && args.nsamples_y == 1));
+ dst_float = dst_f32.alloc(args.ncols_dst * args.nrows_dst);
+ }
+
+ if (dst_float) {
+ run(dst_float, block_nums, tmp_fixup);
+ } else if (args.dst.type == GGML_TYPE_BF16) {
+ run((nv_bfloat16 *) args.dst.p, block_nums, tmp_fixup);
+ }
+#ifdef GGML_MMQ_F16_DST
+ else {
+ run((half *) args.dst.p, block_nums, tmp_fixup);
+ }
+#endif // GGML_MMQ_F16_DST
+
+ if (fixup) {
+ CUDA_CHECK(cudaGetLastError());
+ const dim3 block_nums_fixup(block_nums.x, config.I/warp_size, 1);
+ const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
+ mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
+ (args.ids_dst, args.expert_bounds, dst_float, tmp_fixup, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
+ args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
+ ntx_fd);
+ }
+
+ if (via_f32) {
+ CUDA_CHECK(cudaGetLastError());
+ const dim3 block_nums_store(args.ncols_dst, (args.nrows_x + 255) / 256, 1);
+ if (args.dst.type == GGML_TYPE_BF16) {
+ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (nv_bfloat16 *) args.dst.p, args.nrows_x, args.nrows_dst);
+ } else {
+ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (half *) args.dst.p, args.nrows_x, args.nrows_dst);
+ }
+ }
+ };
+
+ if (!config.stream_k) {
+ launch(block_nums_xy_tiling, false, nullptr);
return;
}
@@ -1441,7 +1561,8 @@
const int ntiles_dst = ntx * nty * ntzw;
const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
- const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1);
+ // The fixup kernel does not read tile_map, so a flat MoE grid always runs one tile per block.
+ const dim3 block_nums_stream_k(args.tile_map || (GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90) ? ntiles_dst : nsm, 1, 1);
GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
@@ -1453,25 +1574,7 @@
tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);
}
- const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1);
- const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
-
- mul_mat_q<type, J, fallback, prec_src1><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale,
- blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
- channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
- sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
- ntx_fd);
-
- if (!fixup_needed) {
- return;
- }
-
- CUDA_CHECK(cudaGetLastError());
- mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
- (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
- args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
- ntx_fd);
+ launch(block_nums_stream_k, fixup_needed, tmp_fixup.ptr);
}
template <ggml_type type, bool fallback, ggml_prec prec_src1>
@@ -1564,6 +1667,7 @@
extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
// -----------------------------------------
extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ1_M);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
@@ -1579,7 +1683,9 @@
// -------------------------------------------------------------------------------------------------------------------------
+// FreeToken: under ids, ncols_max > 0 bounds the tokens any one expert gets and sizes the tile grid; otherwise the grid is flat, see ggml_cuda_launch_mm_ids_tile_map
void ggml_cuda_mul_mat_q(
- ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
+ int64_t ncols_max = -1);
bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -1,4 +1,5 @@
#include "mmvq.cuh"
+#include "convert.cuh"
#include "quantize.cuh"
#include "unary.cuh"
#include "vecdotq.cuh"
@@ -596,10 +597,11 @@
return 1;
}
-template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false>
+// FreeToken: dst_t is float, nv_bfloat16 or half
+template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false, typename dst_t = float>
__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1)
static __global__ void mul_mat_vec_q(
- const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
+ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst_ptr,
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
@@ -608,7 +610,7 @@
const void * GGML_CUDA_RESTRICT vx = vx_ptr;
const void * GGML_CUDA_RESTRICT vy = vy_ptr;
const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
- float * GGML_CUDA_RESTRICT dst = dst_ptr;
+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;
constexpr int qk = ggml_cuda_type_traits<type>::qk;
constexpr int qi = ggml_cuda_type_traits<type>::qi;
@@ -629,7 +631,9 @@
const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y;
if (shared_expert) {
vx = fusion.shared_up;
- dst = fusion.shared_dst;
+ if constexpr (std::is_same_v<dst_t, float>) {
+ dst = fusion.shared_dst;
+ }
stride_col_dst = fusion.shared_stride_col_dst;
}
@@ -702,10 +706,10 @@
}
if constexpr (type == GGML_TYPE_NVFP4) {
if (use_scale) {
- x_scales = x_scale[ids ? channel_x : 0];
+ x_scales = x_scale[(ids ? channel_x : 0) * fusion.x_scale_stride];
}
if (use_gate_scale) {
- gate_scales = gate_scale[ids ? channel_x : 0];
+ gate_scales = gate_scale[(ids ? channel_x : 0) * fusion.gate_scale_stride];
}
}
}
@@ -837,7 +841,7 @@
}
}
}
- dst[j*stride_col_dst + i] = result;
+ dst[j*stride_col_dst + i] = ggml_cuda_cast<dst_t>(result);
}
}
}
@@ -854,11 +858,11 @@
// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)
// Block: (warp_size, ncols_dst) - each warp handles one token independently.
// No shared memory reduction needed since each warp works alone.
-template <ggml_type type, int c_rows_per_block, bool has_fusion = false>
+template <ggml_type type, int c_rows_per_block, bool has_fusion = false, typename dst_t = float>
__launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)
static __global__ void mul_mat_vec_q_moe(
const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion,
- float * dst_ptr,
+ dst_t * dst_ptr,
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst,
const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
@@ -866,7 +870,7 @@
const void * GGML_CUDA_RESTRICT vx = vx_ptr;
const void * GGML_CUDA_RESTRICT vy = vy_ptr;
const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
- float * GGML_CUDA_RESTRICT dst = dst_ptr;
+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;
constexpr int qk = ggml_cuda_type_traits<type>::qk;
constexpr int qi = ggml_cuda_type_traits<type>::qi;
@@ -878,7 +882,9 @@
const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;
if (shared_expert) {
vx = fusion.shared_up;
- dst = fusion.shared_dst;
+ if constexpr (std::is_same_v<dst_t, float>) {
+ dst = fusion.shared_dst;
+ }
stride_col_dst = fusion.shared_stride_col_dst;
}
@@ -963,7 +969,7 @@
if constexpr (type == GGML_TYPE_NVFP4) {
if (x_scale) {
- result *= x_scale[channel_x];
+ result *= x_scale[channel_x * fusion.x_scale_stride];
}
}
if (x_bias) {
@@ -973,7 +979,7 @@
float gate_value = tmp_gate[threadIdx.x];
if constexpr (type == GGML_TYPE_NVFP4) {
if (gate_scale) {
- gate_value *= gate_scale[channel_x];
+ gate_value *= gate_scale[channel_x * fusion.gate_scale_stride];
}
}
if (gate_bias) {
@@ -998,7 +1004,7 @@
}
}
}
- dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = result;
+ dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = ggml_cuda_cast<dst_t>(result);
}
if constexpr (!has_fusion) {
@@ -1020,9 +1026,9 @@
return {block_nums, block_dims};
}
-template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false>
+template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false, typename dst_t>
static void mul_mat_vec_q_switch_fusion(
- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
@@ -1035,7 +1041,7 @@
if constexpr (c_ncols_dst == 1) {
if (has_fusion) {
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
- ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters>, launch_params,
+ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters, dst_t>, launch_params,
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
@@ -1046,15 +1052,15 @@
GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1");
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
- ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters>, launch_params,
+ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters, dst_t>, launch_params,
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
}
-template <ggml_type type>
+template <ggml_type type, typename dst_t>
static void mul_mat_vec_q_moe_launch(
- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,
const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
@@ -1071,13 +1077,13 @@
fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
if (has_fusion) {
- ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true>, launch_params,
+ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true, dst_t>, launch_params,
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
stride_row_x, stride_col_y, stride_col_dst,
stride_channel_x, stride_channel_y, stride_channel_dst,
ncols_dst, ids_stride);
} else {
- ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false>, launch_params,
+ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false, dst_t>, launch_params,
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
stride_row_x, stride_col_y, stride_col_dst,
stride_channel_x, stride_channel_y, stride_channel_dst,
@@ -1085,9 +1091,9 @@
}
}
-template <ggml_type type>
+template <ggml_type type, typename dst_t>
static void mul_mat_vec_q_switch_ncols_dst(
- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
const int ncols_x, const int nrows_x, const int ncols_dst,
const int stride_row_x, const int stride_col_y, const int stride_col_dst,
const int nchannels_x, const int nchannels_y, const int nchannels_dst,
@@ -1277,8 +1283,10 @@
GGML_ABORT("fatal error");
}
}
+
+template <typename dst_t>
static void mul_mat_vec_q_switch_type(
- const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
+ const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
const int ncols_x, const int nrows_x, const int ncols_dst,
const int stride_row_x, const int stride_col_y, const int stride_col_dst,
const int nchannels_x, const int nchannels_y, const int nchannels_dst,
@@ -1431,9 +1439,10 @@
void ggml_cuda_mul_mat_vec_q(
ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
- const ggml_cuda_mm_fusion_args_host * fusion) {
- GGML_ASSERT( src1->type == GGML_TYPE_F32);
- GGML_ASSERT( dst->type == GGML_TYPE_F32);
+ const ggml_cuda_mm_fusion_args_host * fusion, const void * src1_q8_1_pre) {
+ // FreeToken: activations and results may also be bf16 or fp16
+ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);
GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
GGML_TENSOR_BINARY_OP_LOCALS;
@@ -1498,17 +1507,20 @@
GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);
fusion_local.gate_bias = fusion->gate_bias->data;
}
+ // FreeToken: a scale may be a strided vector, read with its element stride
if (fusion->x_scale) {
GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);
- GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));
+ GGML_ASSERT(ggml_nelements(fusion->x_scale) == fusion->x_scale->ne[0] && fusion->x_scale->nb[0] % sizeof(float) == 0);
GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));
fusion_local.x_scale = fusion->x_scale->data;
+ fusion_local.x_scale_stride = fusion->x_scale->nb[0] / sizeof(float);
}
if (fusion->gate_scale) {
GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);
- GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));
+ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == fusion->gate_scale->ne[0] && fusion->gate_scale->nb[0] % sizeof(float) == 0);
GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));
fusion_local.gate_scale = fusion->gate_scale->data;
+ fusion_local.gate_scale_stride = fusion->gate_scale->nb[0] / sizeof(float);
}
fusion_local.glu_op = fusion->glu_op;
fusion_local.glu_limit = fusion->glu_limit;
@@ -1526,12 +1538,22 @@
}
const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);
- ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
- {
+ // FreeToken: src1_q8_1_pre is src1 already quantized by quantize_row_q8_1_cuda with this padding and layout
+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool());
+ const void * src1_q8 = src1_q8_1_pre;
+ if (src1_q8 == nullptr) {
+ src1_q8_1.alloc(ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
+ src1_q8 = src1_q8_1.get();
const int64_t s11 = src1->nb[1] / ts_src1;
const int64_t s12 = src1->nb[2] / ts_src1;
const int64_t s13 = src1->nb[3] / ts_src1;
- quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+ if (src1->type == GGML_TYPE_BF16) {
+ quantize_row_q8_1_cuda((const nv_bfloat16 *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+ } else if (src1->type == GGML_TYPE_F16) {
+ quantize_row_q8_1_cuda((const half *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+ } else {
+ quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+ }
}
const int64_t s01 = src0->nb[1] / ts_src0;
@@ -1556,11 +1578,23 @@
const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
- mul_mat_vec_q_switch_type(
- src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00,
- ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
- ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
- ne03, ne3, s03, s13, s3, ids_stride, stream);
+ const auto launch = [&](auto * dst_ptr) {
+ mul_mat_vec_q_switch_type(
+ src0->data, src0->type, src1_q8, ids_d, fusion_local, dst_ptr, ne00,
+ ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
+ ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
+ ne03, ne3, s03, s13, s3, ids_stride, stream);
+ };
+ if (dst->type == GGML_TYPE_BF16) {
+ // FreeToken: the fused shared expert writes an fp32 shared_dst
+ GGML_ASSERT(!fusion || !fusion->shared_up);
+ launch((nv_bfloat16 *) dst->data);
+ } else if (dst->type == GGML_TYPE_F16) {
+ GGML_ASSERT(!fusion || !fusion->shared_up);
+ launch((half *) dst->data);
+ } else {
+ launch(dst_d);
+ }
}
void ggml_cuda_op_mul_mat_vec_q(
--- a/ggml/src/ggml-cuda/mmvq.cuh
+++ b/ggml/src/ggml-cuda/mmvq.cuh
@@ -8,8 +8,10 @@
// based on the quantization type and GPU architecture (compute capability).
int get_mmvq_mmid_max_batch(ggml_type type, int cc);
+// FreeToken: src1_q8_1_pre skips the internal quantization of src1
void ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx,
- const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr);
+ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr,
+ const void * src1_q8_1_pre = nullptr);
void ggml_cuda_op_mul_mat_vec_q(
ggml_backend_cuda_context & ctx,
--- a/ggml/src/ggml-cuda/quantize.cu
+++ b/ggml/src/ggml-cuda/quantize.cu
@@ -1,4 +1,5 @@
#include "quantize.cuh"
+#include "convert.cuh"
#include <cstdint>
#if defined(BLACKWELL_MMA_AVAILABLE)
@@ -50,13 +51,14 @@
#endif // CUDART_VERSION >= 12080
#endif // defined(BLACKWELL_MMA_AVAILABLE)
+template <typename src_t>
__launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)
static __global__ void quantize_q8_1(
- const float * x_ptr, void * vy_ptr,
+ const src_t * x_ptr, void * vy_ptr,
const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
const int64_t ne0, const uint32_t ne1, const uint3 ne2) {
ggml_cuda_pdl_lc();
- const float * GGML_CUDA_RESTRICT x = x_ptr;
+ const src_t * GGML_CUDA_RESTRICT x = x_ptr;
void * GGML_CUDA_RESTRICT vy = vy_ptr;
const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
@@ -81,7 +83,7 @@
const int64_t iqs = i_cont % QK8_1; // quant index
ggml_cuda_pdl_sync();
- const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f;
+ const float xi = i0 < ne00 ? ggml_cuda_cast<float>(x[i03*s03 + i02*s02 + i01*s01 + i00]) : 0.0f;
float amax = fabsf(xi);
float sum = xi;
@@ -452,10 +454,29 @@
GGML_UNUSED(n_expert_used);
}
+// FreeToken: 4 consecutive activations as float, from fp32, bf16 or fp16
+static __device__ __forceinline__ float4 load_float4(const float * x) {
+ return *(const float4 *) x;
+}
+
+static __device__ __forceinline__ float4 load_float4(const nv_bfloat16 * x) {
+ const uint2 v = *(const uint2 *) x;
+ const float2 lo = __bfloat1622float2(*(const nv_bfloat162 *) &v.x);
+ const float2 hi = __bfloat1622float2(*(const nv_bfloat162 *) &v.y);
+ return make_float4(lo.x, lo.y, hi.x, hi.y);
+}
+
+static __device__ __forceinline__ float4 load_float4(const half * x) {
+ const uint2 v = *(const uint2 *) x;
+ const float2 lo = __half22float2(*(const half2 *) &v.x);
+ const float2 hi = __half22float2(*(const half2 *) &v.y);
+ return make_float4(lo.x, lo.y, hi.x, hi.y);
+}
+
// scatter: grid over tokens, quantize once, write to all the token's compact rows
-template <mmq_q8_1_ds_layout ds_layout, bool scatter>
+template <mmq_q8_1_ds_layout ds_layout, bool scatter, typename src_t>
static __global__ void quantize_mmq_q8_1(
- const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,
+ const src_t * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,
const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) {
@@ -481,14 +502,13 @@
base_idx = i3*s03 + i2*s02 + i01*s01;
}
- const float4 * x4 = (const float4 *) x;
block_q8_1_mmq * y = (block_q8_1_mmq *) vy;
const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel
const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block
// Load 4 floats per thread and calculate max. abs. value between them:
- const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ const float4 xi = i0 < ne00 ? load_float4(x + base_idx + i00) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
float amax = fabsf(xi.x);
amax = fmaxf(amax, fabsf(xi.y));
amax = fmaxf(amax, fabsf(xi.z));
@@ -554,8 +574,9 @@
GGML_UNUSED(n_expert_used);
}
+template <typename src_t>
void quantize_row_q8_1_cuda(
- const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
+ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,
const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
GGML_ASSERT(!ids);
@@ -567,12 +588,13 @@
const dim3 num_blocks(block_num_x, ne1, ne2*ne3);
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream);
- ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);
+ ggml_cuda_kernel_launch(quantize_q8_1<src_t>, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);
GGML_UNUSED(type_src0);
}
+template <typename src_t>
void quantize_mmq_q8_1_cuda(
- const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
+ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,
const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
GGML_ASSERT(ne00 % 4 == 0);
@@ -584,15 +606,15 @@
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
switch (mmq_get_q8_1_ds_layout(type_src0)) {
case MMQ_Q8_1_DS_LAYOUT_D4:
- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false>
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false, src_t>
<<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
break;
case MMQ_Q8_1_DS_LAYOUT_DS4:
- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false>
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false, src_t>
<<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
break;
case MMQ_Q8_1_DS_LAYOUT_D2S6:
- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false>
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false, src_t>
<<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
break;
default:
@@ -601,8 +623,9 @@
}
// scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)
+template <typename src_t>
void quantize_scatter_mmq_q8_1_cuda(
- const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
+ const src_t * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
const int64_t ne00, const int64_t stride_token, const int64_t ne0,
const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) {
GGML_ASSERT(ne00 % 4 == 0);
@@ -613,15 +636,15 @@
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
switch (mmq_get_q8_1_ds_layout(type_src0)) {
case MMQ_Q8_1_DS_LAYOUT_D4:
- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true><<<num_blocks, block_size, 0, stream>>>(
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true, src_t><<<num_blocks, block_size, 0, stream>>>(
x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
break;
case MMQ_Q8_1_DS_LAYOUT_DS4:
- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true><<<num_blocks, block_size, 0, stream>>>(
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true, src_t><<<num_blocks, block_size, 0, stream>>>(
x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
break;
case MMQ_Q8_1_DS_LAYOUT_D2S6:
- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true><<<num_blocks, block_size, 0, stream>>>(
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true, src_t><<<num_blocks, block_size, 0, stream>>>(
x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
break;
default:
@@ -692,3 +715,15 @@
quantize_mmq_mxfp4<false><<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
}
}
+
+#define FREETOKEN_QUANTIZE_INSTANCES(src_t) \
+ template void quantize_row_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \
+ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \
+ template void quantize_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \
+ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \
+ template void quantize_scatter_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, \
+ int64_t, int64_t, int64_t, int64_t, int, cudaStream_t);
+
+FREETOKEN_QUANTIZE_INSTANCES(float)
+FREETOKEN_QUANTIZE_INSTANCES(nv_bfloat16)
+FREETOKEN_QUANTIZE_INSTANCES(half)
--- a/ggml/src/ggml-cuda/quantize.cuh
+++ b/ggml/src/ggml-cuda/quantize.cuh
@@ -16,13 +16,16 @@
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+// FreeToken: src_t is float, nv_bfloat16 or half
+template <typename src_t>
void quantize_row_q8_1_cuda(
- const float * x, const int32_t * ids, void * vy,
+ const src_t * x, const int32_t * ids, void * vy,
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+template <typename src_t>
void quantize_mmq_q8_1_cuda(
- const float * x, const int32_t * ids, void * vy,
+ const src_t * x, const int32_t * ids, void * vy,
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
@@ -57,7 +60,8 @@
int n_expert_used,
cudaStream_t stream);
-void quantize_scatter_mmq_q8_1_cuda(const float * x,
+template <typename src_t>
+void quantize_scatter_mmq_q8_1_cuda(const src_t * x,
const int32_t * ids_src1_inv,
void * vy,
ggml_type type_src0,
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_m.cu
@@ -0,0 +1,5 @@
+// FreeToken: not generated, llama.cpp has no IQ1_M MMQ.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ1_M);