LLM inference in C/C++
C++ ggml
CUDA: add Q2_0 support (#25707)
Pasha Khosravi9b2a088819cd
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuhindex fa2dd26202..33be16dc5c 100644--- a/ggml/src/ggml-cuda/common.cuh+++ b/ggml/src/ggml-cuda/common.cuh@@ -977,6 +977,13 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q1_0> {static constexpr int qi = QI1_0;};+template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q2_0> {+ static constexpr int qk = QK2_0;+ static constexpr int qr = QR2_0;+ static constexpr int qi = QI2_0;+};+template<>struct ggml_cuda_type_traits<GGML_TYPE_Q4_0> {static constexpr int qk = QK4_0;diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cuindex 946e02af50..360c614a44 100644--- a/ggml/src/ggml-cuda/convert.cu+++ b/ggml/src/ggml-cuda/convert.cu@@ -459,6 +459,8 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {switch (type) {case GGML_TYPE_Q1_0:return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;case GGML_TYPE_Q4_0:return dequantize_row_q4_0_cuda;case GGML_TYPE_Q4_1:@@ -514,6 +516,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {switch (type) {case GGML_TYPE_Q1_0:return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;case GGML_TYPE_Q4_0:return dequantize_row_q4_0_cuda;case GGML_TYPE_Q4_1:@@ -572,6 +576,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {switch (type) {case GGML_TYPE_Q1_0:return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;case GGML_TYPE_Q4_0:return dequantize_row_q4_0_cuda;case GGML_TYPE_Q4_1:@@ -629,6 +635,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {return convert_unary_cuda<float>;case GGML_TYPE_Q1_0:return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;case GGML_TYPE_Q4_0:return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;case GGML_TYPE_Q4_1:@@ -652,6 +660,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) {return convert_unary_cuda<float, nv_bfloat16>;case GGML_TYPE_Q1_0:return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;case GGML_TYPE_Q4_0:return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;case GGML_TYPE_Q4_1:@@ -675,6 +685,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {return convert_unary_cuda<half, float>;case GGML_TYPE_Q1_0:return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;case GGML_TYPE_Q4_0:return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;case GGML_TYPE_Q4_1:diff --git a/ggml/src/ggml-cuda/dequantize.cuh b/ggml/src/ggml-cuda/dequantize.cuhindex 8ab5ad8e74..651524be7b 100644--- a/ggml/src/ggml-cuda/dequantize.cuh+++ b/ggml/src/ggml-cuda/dequantize.cuh@@ -23,6 +23,26 @@ static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const inv.y = (2*bit_1 - 1) * d;}+static __device__ __forceinline__ void dequantize_q2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q2_0 * x = (const block_q2_0 *) vx;++ const float d = x[ib].d;++ // Q2_0: 2 bits per element, 4 elements per byte.+ // Stored code c in {0,1,2,3} maps to symbol s = c - 1 in {-1, 0, +1, +2}.+ const int byte_index_0 = iqs / 4;+ const int bit_offset_0 = (iqs % 4) * 2;++ const int byte_index_1 = (iqs + 1) / 4;+ const int bit_offset_1 = ((iqs + 1) % 4) * 2;++ const int c0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 0x3;+ const int c1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 0x3;++ v.x = (c0 - 1) * d;+ v.y = (c1 - 1) * d;+}+static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){const block_q4_0 * x = (const block_q4_0 *) vx;diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cuindex a9ec4f697c..6b36b8fbc1 100644--- a/ggml/src/ggml-cuda/getrows.cu+++ b/ggml/src/ggml-cuda/getrows.cu@@ -320,6 +320,10 @@ static void ggml_cuda_get_rows_switch_src0_type(get_rows_cuda_q<QK1_0, QR1_0, dequantize_q1_0>(src0_d, src1_d, dst_d,ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);break;+ case GGML_TYPE_Q2_0:+ get_rows_cuda_q<QK2_0, QR2_0, dequantize_q2_0>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;case GGML_TYPE_Q4_0:get_rows_cuda_q<QK4_0, QR4_0, dequantize_q4_0>(src0_d, src1_d, dst_d,ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex e73a7b8906..d22b5657d9 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -4802,6 +4802,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_TYPE_F32:case GGML_TYPE_F16:case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -4840,6 +4841,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_TYPE_BF16:case GGML_TYPE_I32:case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:diff --git a/ggml/src/ggml-cuda/mmq-config-ampere.cuh b/ggml/src/ggml-cuda/mmq-config-ampere.cuhindex 0037bac3d0..9f9fd19738 100644--- a/ggml/src/ggml-cuda/mmq-config-ampere.cuh+++ b/ggml/src/ggml-cuda/mmq-config-ampere.cuh@@ -16,6 +16,23 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_confCASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);diff --git a/ggml/src/ggml-cuda/mmq-config-cdna.cuh b/ggml/src/ggml-cuda/mmq-config-cdna.cuhindex 46ec6aa9d5..4a8d89f720 100644--- a/ggml/src/ggml-cuda/mmq-config-cdna.cuh+++ b/ggml/src/ggml-cuda/mmq-config-cdna.cuh@@ -7,6 +7,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_confCASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);diff --git a/ggml/src/ggml-cuda/mmq-config-pascal.cuh b/ggml/src/ggml-cuda/mmq-config-pascal.cuhindex 8f0faac889..e7d4a9a3fc 100644--- a/ggml/src/ggml-cuda/mmq-config-pascal.cuh+++ b/ggml/src/ggml-cuda/mmq-config-pascal.cuh@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_confCASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);diff --git a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh b/ggml/src/ggml-cuda/mmq-config-rdna2.cuhindex de4db0a3db..8324d9e1a8 100644--- a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh+++ b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_confCASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuhindex e420a32f0c..180b2d9370 100644--- a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh+++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_confCASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3.cuhindex 1226230609..676f27fea4 100644--- a/ggml/src/ggml-cuda/mmq-config-rdna3.cuh+++ b/ggml/src/ggml-cuda/mmq-config-rdna3.cuh@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_confCASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);diff --git a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh b/ggml/src/ggml-cuda/mmq-config-rdna4.cuhindex a224ecafc3..9293d9d558 100644--- a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh+++ b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_confCASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuhindex 7fb242096e..8c9070c6be 100644--- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh+++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh@@ -95,6 +95,103 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_}}+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0;+ constexpr int threads_per_row = blocks_per_iter * QI2_0;+ constexpr int nrows = warp_size / threads_per_row;+ constexpr int scale_entries_per_block = QK2_0 / QK8_1;+ constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;++ const int txi = threadIdx.x % threads_per_row;+ const int kbx = txi / QI2_0;+ const int kqsx = txi % QI2_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx;+ // Each 32-element chunk occupies 8 bytes of qs (32 elements * 2 bits = 64 bits)+ const int qs_offset = 8*kqsx;+ const int qs0 = bxi->qs[qs_offset + 0] | (bxi->qs[qs_offset + 1] << 8) |+ (bxi->qs[qs_offset + 2] << 16) | (bxi->qs[qs_offset + 3] << 24);+ const int qs1 = bxi->qs[qs_offset + 4] | (bxi->qs[qs_offset + 5] << 8) |+ (bxi->qs[qs_offset + 6] << 16) | (bxi->qs[qs_offset + 7] << 24);++ // Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8s in {-1,0,1,2}.+ int unpacked_bytes[8];+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int shift = j * 8;+ const int codes = (qs0 >> shift) & 0xFF;+ const int c0 = ((codes >> 0) & 0x3) - 1;+ const int c1 = ((codes >> 2) & 0x3) - 1;+ const int c2 = ((codes >> 4) & 0x3) - 1;+ const int c3 = ((codes >> 6) & 0x3) - 1;+ unpacked_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);+ }+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int shift = j * 8;+ const int codes = (qs1 >> shift) & 0xFF;+ const int c0 = ((codes >> 0) & 0x3) - 1;+ const int c1 = ((codes >> 2) & 0x3) - 1;+ const int c2 = ((codes >> 4) & 0x3) - 1;+ const int c3 = ((codes >> 6) & 0x3) - 1;+ unpacked_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);+ }++ const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + dst_offset + j] = unpacked_bytes[j];+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j] = unpacked_bytes[j];+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }++ const int ksx = threadIdx.x % scale_entries_per_row;+ const int scale_block = ksx / scale_entries_per_block;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps) {+ int i = i0 + threadIdx.y;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + ksx] = bxi->d;+#else+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0(const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {constexpr int warp_size = ggml_cuda_get_physical_warp_size();diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cuindex 8cf1a3eb94..707437ea3e 100644--- a/ggml/src/ggml-cuda/mmq.cu+++ b/ggml/src/ggml-cuda/mmq.cu@@ -10,6 +10,9 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, concase GGML_TYPE_Q1_0:mul_mat_q_case<GGML_TYPE_Q1_0>(ctx, args, stream);break;+ case GGML_TYPE_Q2_0:+ mul_mat_q_case<GGML_TYPE_Q2_0>(ctx, args, stream);+ break;case GGML_TYPE_Q4_0:mul_mat_q_case<GGML_TYPE_Q4_0>(ctx, args, stream);break;@@ -262,6 +265,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_tswitch (type) {case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuhindex 5f30f5f6bc..2eb15fdfad 100644--- a/ggml/src/ggml-cuda/mmq.cuh+++ b/ggml/src/ggml-cuda/mmq.cuh@@ -60,6 +60,7 @@ static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected bstatic mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {switch (type_x) {case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:return MMQ_Q8_1_DS_LAYOUT_D4;case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:@@ -385,6 +386,7 @@ static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type,static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) {switch (type) {case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0;case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;@@ -542,6 +544,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q2_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q2_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);case GGML_TYPE_Q4_0:return ggml_cuda_mmq_util_funcs(VDR_Q4_0_Q8_1_MMQ,@@ -700,6 +708,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q2_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);case GGML_TYPE_Q4_0:return ggml_cuda_mmq_util_funcs(-1,@@ -1550,6 +1564,7 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudatemplate void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \extern DECL_MMQ_CASE(GGML_TYPE_Q1_0);+extern DECL_MMQ_CASE(GGML_TYPE_Q2_0);extern DECL_MMQ_CASE(GGML_TYPE_Q4_0);extern DECL_MMQ_CASE(GGML_TYPE_Q4_1);extern DECL_MMQ_CASE(GGML_TYPE_Q5_0);diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cuindex e18ada5377..0589e65bdf 100644--- a/ggml/src/ggml-cuda/mmvq.cu+++ b/ggml/src/ggml-cuda/mmvq.cu@@ -10,6 +10,7 @@ typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {switch (type) {case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1;+ case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1;case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1;case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1;case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1;@@ -38,6 +39,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type)static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {switch (type) {case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ;+ case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ;case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ;case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ;case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ;@@ -1010,6 +1012,12 @@ static void mul_mat_vec_q_switch_type(nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);break;+ case GGML_TYPE_Q2_0:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_0>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;case GGML_TYPE_Q4_0:mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_0>(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.pyindex 614b1566c7..d7cd271675 100755--- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py+++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py@@ -36,6 +36,7 @@ SOURCE_FATTN_MMA_CASE = "DECL_FATTN_MMA_F16_CASE({head_size_kq}, {head_size_v},TYPES_MMQ = ["GGML_TYPE_Q1_0",+ "GGML_TYPE_Q2_0","GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0","GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K","GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S",diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cunew file mode 100644index 0000000000..750180e330--- /dev/null+++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q2_0);diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuhindex b9932bce9c..36d89c89e3 100644--- a/ggml/src/ggml-cuda/vecdotq.cuh+++ b/ggml/src/ggml-cuda/vecdotq.cuh@@ -109,6 +109,9 @@ static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) {#define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism#define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block+#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism+#define VDR_Q2_0_Q8_1_MMQ 2 // Q2_0 group 64: 128 bits (4 ints) per block, 2 32-element chunks+#define VDR_Q4_0_Q8_1_MMVQ 2#define VDR_Q4_0_Q8_1_MMQ 4@@ -722,6 +725,64 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1(return d1 * d8 * sumi;}+static __device__ __forceinline__ float vec_dot_q2_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx;++ // Q2_0 (group 64): 64 elements with ONE scale, 2 bits per element (4 elements per byte)+ // Q8_1: 32 elements per block with individual scales+ // iqs selects which of the 2 chunks of 32 elements to process (0-1)++ const float d2 = bq2_0->d;++ // Process only the chunk specified by iqs+ const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;++ // Load 64 bits (8 bytes) for this chunk from Q2_0: bytes [8*iqs, 8*iqs+8)+ const int offset = iqs * 8;+ const int v0 = bq2_0->qs[offset + 0] | (bq2_0->qs[offset + 1] << 8) |+ (bq2_0->qs[offset + 2] << 16) | (bq2_0->qs[offset + 3] << 24);+ const int v1 = bq2_0->qs[offset + 4] | (bq2_0->qs[offset + 5] << 8) |+ (bq2_0->qs[offset + 6] << 16) | (bq2_0->qs[offset + 7] << 24);++ // Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8 symbols in {-1,0,1,2}.+ // Stored code c in {0,1,2,3} -> symbol s = c - 1.+ int vi_bytes[8];+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int shift = j * 8;+ const int codes = (v0 >> shift) & 0xFF;+ const int c0 = ((codes >> 0) & 0x3) - 1;+ const int c1 = ((codes >> 2) & 0x3) - 1;+ const int c2 = ((codes >> 4) & 0x3) - 1;+ const int c3 = ((codes >> 6) & 0x3) - 1;+ vi_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);+ }+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int shift = j * 8;+ const int codes = (v1 >> shift) & 0xFF;+ const int c0 = ((codes >> 0) & 0x3) - 1;+ const int c1 = ((codes >> 2) & 0x3) - 1;+ const int c2 = ((codes >> 4) & 0x3) - 1;+ const int c3 = ((codes >> 6) & 0x3) - 1;+ vi_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);+ }++ // Compute dot product for this 32-element chunk+ int sumi = 0;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const int u = get_int_b4(bq8_1_chunk->qs, j);+ sumi = ggml_cuda_dp4a(vi_bytes[j], u, sumi);+ }++ // Apply Q2_0's single scale and this chunk's Q8_1 scale+ const float d8 = __low2float(bq8_1_chunk->ds);+ return d2 * d8 * sumi;+}+static __device__ __forceinline__ float vec_dot_q4_0_q8_1(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex d0eb173a8f..ef18ecef8a 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -8013,6 +8013,7 @@ static const ggml_type other_types[] = {GGML_TYPE_Q5_0, GGML_TYPE_Q5_1,GGML_TYPE_Q8_0,GGML_TYPE_Q1_0,+ GGML_TYPE_Q2_0,GGML_TYPE_Q2_K, GGML_TYPE_Q3_K,GGML_TYPE_Q5_K,GGML_TYPE_Q6_K,@@ -9563,6 +9564,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_Q4_0));test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q1_0));test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_F16));+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q2_0));+ test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0));+ test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0));+ test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16));// large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix// stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG).