LLM inference in C/C++
C++ ggml
Add Q2_0 quantization: type definition and CPU backend (#24448)
Pasha Khosravibec4772f6a25
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.hindex d6807b6dd4..ac133665d9 100644--- a/ggml/include/ggml.h+++ b/ggml/include/ggml.h@@ -429,7 +429,8 @@ extern "C" {GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)GGML_TYPE_Q1_0 = 41,- GGML_TYPE_COUNT = 42,+ GGML_TYPE_Q2_0 = 42,+ GGML_TYPE_COUNT = 43,};// precision@@ -473,6 +474,7 @@ extern "C" {GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensorsGGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensorsGGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors};// available tensor operations:diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.hindex a51b37dcdb..83f9118da8 100644--- a/ggml/src/ggml-common.h+++ b/ggml/src/ggml-common.h@@ -96,6 +96,9 @@ typedef sycl::half2 ggml_half2;#define QI1_0 (QK1_0 / 32)#define QR1_0 1+#define QI2_0 (QK2_0 / 32)+#define QR2_0 1+#define QI4_0 (QK4_0 / (4 * QR4_0))#define QR4_0 2@@ -181,6 +184,13 @@ typedef struct {} block_q1_0;static_assert(sizeof(block_q1_0) == sizeof(ggml_half) + QK1_0 / 8, "wrong q1_0 block size/padding");+#define QK2_0 64+typedef struct {+ ggml_half d; // delta (scale)+ uint8_t qs[QK2_0 / 4]; // 2 bits per element+} block_q2_0;+static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding");+#define QK4_0 32typedef struct {ggml_half d; // deltadiff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.hindex da2ac98a48..152e0bac99 100644--- a/ggml/src/ggml-cpu/arch-fallback.h+++ b/ggml/src/ggml-cpu/arch-fallback.h@@ -17,6 +17,7 @@#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K@@ -82,6 +83,7 @@#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)// quants.c+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0// repack.cpp#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4@@ -113,6 +115,7 @@#define quantize_row_q8_K_generic quantize_row_q8_K#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K@@ -162,6 +165,7 @@#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0// repack.cpp#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8@@ -202,6 +206,7 @@#elif defined(__riscv)// quants.c#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0// repack.cpp#define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4@@ -243,6 +248,7 @@#define quantize_row_q8_K_generic quantize_row_q8_K#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K@@ -306,6 +312,7 @@#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0+#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0// repack.cpp#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.cindex 4453667976..636d7be124 100644--- a/ggml/src/ggml-cpu/arch/arm/quants.c+++ b/ggml/src/ggml-cpu/arch/arm/quants.c@@ -219,6 +219,80 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi#endif}+void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK2_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ float sumf = 0.0f;++#if defined(__ARM_NEON)+ // Replicate pattern: each byte repeated 4 times+ static const uint8_t tbl_idx_lo[16] = {0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3};+ static const uint8_t tbl_idx_hi[16] = {4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7};+ // Right-shift amounts: 0,2,4,6 repeated for each group of 4+ static const int8_t shift_vals[16] = {0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6};++ const uint8x16_t idx_lo = vld1q_u8(tbl_idx_lo);+ const uint8x16_t idx_hi = vld1q_u8(tbl_idx_hi);+ const int8x16_t shifts = vld1q_s8(shift_vals);+ const uint8x16_t mask2 = vdupq_n_u8(0x03);+ const int8x16_t one = vdupq_n_s8(1);++ float32x4_t sumv = vdupq_n_f32(0.0f);++ for (int i = 0; i < nb; i++) {+ const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d);++ // group 64: one Q2_0 block (64 weights) maps to two Q8_0 blocks (2 * 32 = 64)+ for (int k = 0; k < 2; k++) {+ const block_q8_0 * GGML_RESTRICT yb = &y[i * 2 + k];+ const float d1 = GGML_CPU_FP16_TO_FP32(yb->d);++ // Load 8 bytes of packed 2-bit values+ const uint8x8_t raw = vld1_u8(&x[i].qs[k * 8]);+ const uint8x16_t raw16 = vcombine_u8(raw, raw);++ // First 16 elements: replicate bytes 0-3, shift, mask, subtract 1+ uint8x16_t bytes0 = vqtbl1q_u8(raw16, idx_lo);+ int8x16_t qv0 = vsubq_s8(+ vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes0, shifts), mask2)),+ one);++ // Second 16 elements: replicate bytes 4-7, shift, mask, subtract 1+ uint8x16_t bytes1 = vqtbl1q_u8(raw16, idx_hi);+ int8x16_t qv1 = vsubq_s8(+ vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes1, shifts), mask2)),+ one);++ // Load Q8_0 values and dot product+ const int8x16_t y0 = vld1q_s8(yb->qs);+ const int8x16_t y1 = vld1q_s8(yb->qs + 16);++ int32x4_t p0 = ggml_vdotq_s32(vdupq_n_s32(0), qv0, y0);+ int32x4_t p1 = ggml_vdotq_s32(p0, qv1, y1);++ sumv = vmlaq_n_f32(sumv, vcvtq_f32_s32(p1), d0 * d1);+ }+ }++ sumf = vaddvq_f32(sumv);+#else+ ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);+ return;+#endif++ *s = sumf;+}void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {const int qk = QK8_0;diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cindex 8a0e5c255f..a82842fcff 100644--- a/ggml/src/ggml-cpu/ggml-cpu.c+++ b/ggml/src/ggml-cpu/ggml-cpu.c@@ -230,6 +230,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {.vec_dot_type = GGML_TYPE_Q8_0,.nrows = 1,},+ [GGML_TYPE_Q2_0] = {+ .from_float = quantize_row_q2_0,+ .vec_dot = ggml_vec_dot_q2_0_q8_0,+ .vec_dot_type = GGML_TYPE_Q8_0,+ .nrows = 1,+ },[GGML_TYPE_Q4_0] = {.from_float = quantize_row_q4_0,.vec_dot = ggml_vec_dot_q4_0_q8_0,diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex c555831ce7..a735efff44 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -665,6 +665,7 @@ void ggml_compute_forward_add(ggml_compute_forward_add_non_quantized(params, dst);} break;case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -1115,6 +1116,7 @@ void ggml_compute_forward_add1(}} break;case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -1245,6 +1247,7 @@ void ggml_compute_forward_acc(case GGML_TYPE_F16:case GGML_TYPE_BF16:case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -4454,6 +4457,7 @@ void ggml_compute_forward_out_prod(switch (src0->type) {case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -4730,6 +4734,7 @@ void ggml_compute_forward_set(case GGML_TYPE_F16:case GGML_TYPE_BF16:case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -4954,6 +4959,7 @@ void ggml_compute_forward_get_rows(switch (src0->type) {case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -5680,6 +5686,7 @@ void ggml_compute_forward_clamp(} break;case GGML_TYPE_BF16:case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.cindex e5f9a4083f..5e36459f8c 100644--- a/ggml/src/ggml-cpu/quants.c+++ b/ggml/src/ggml-cpu/quants.c@@ -26,6 +26,10 @@ void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, inquantize_row_q1_0_ref(x, y, k);}+void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ quantize_row_q2_0_ref(x, y, k);+}+void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {quantize_row_q4_0_ref(x, y, k);}@@ -170,6 +174,53 @@ void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c*s = sumf;}+void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ const int qk = QK2_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;++ float sumf = 0.0f;++ for (int i = 0; i < nb; i++) {+ const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d);++ float sumi = 0.0f;++ // group 64: one Q2_0 block (64 weights) maps to two Q8_0 blocks (2 * 32 = 64)+ for (int k = 0; k < 2; k++) {+ const block_q8_0 * GGML_RESTRICT yb = &y[i * 2 + k];+ const float d1 = GGML_CPU_FP16_TO_FP32(yb->d);+ int sumi_block = 0;++ const uint8_t * GGML_RESTRICT qs = &x[i].qs[k * 8];+ const int8_t * GGML_RESTRICT qy = yb->qs;++ for (int b = 0; b < 8; ++b) {+ const uint8_t byte = qs[b];+ // Extract 4 two-bit values, map {0,1,2,3} -> {-1,0,1,2}+ sumi_block += ((int)((byte >> 0) & 3) - 1) * qy[b*4 + 0];+ sumi_block += ((int)((byte >> 2) & 3) - 1) * qy[b*4 + 1];+ sumi_block += ((int)((byte >> 4) & 3) - 1) * qy[b*4 + 2];+ sumi_block += ((int)((byte >> 6) & 3) - 1) * qy[b*4 + 3];+ }++ sumi += d1 * sumi_block;+ }++ sumf += d0 * sumi;+ }++ *s = sumf;+}void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {const int qk = QK8_0;diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.hindex d4bc87a1c0..93ea7eeffe 100644--- a/ggml/src/ggml-cpu/quants.h+++ b/ggml/src/ggml-cpu/quants.h@@ -13,6 +13,7 @@ extern "C" {// Quantizationvoid quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);@@ -38,6 +39,7 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y,// Dot productvoid ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);@@ -71,6 +73,7 @@ void quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRIvoid quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.cindex 15d231f70c..1ebc50a763 100644--- a/ggml/src/ggml-quants.c+++ b/ggml/src/ggml-quants.c@@ -71,6 +71,44 @@ void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_REST}}+void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK2_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ // Compute scale as max absolute value in the block+ float amax = 0.0f;+ for (int j = 0; j < qk; j++) {+ const float a = fabsf(x[i*qk + j]);+ if (a > amax) amax = a;+ }+ const float d = amax;+ const float id = d > 0.0f ? 1.0f / d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ // Clear quant bytes+ for (int j = 0; j < qk / 4; ++j) {+ y[i].qs[j] = 0;+ }++ // Encode 2-bit values: round(w/d) clamped to [-1, 2], then add 1+ // 00 (-1) = -scale, 01 (0) = 0, 10 (+1) = +scale, 11 (+2) = 2*scale+ for (int j = 0; j < qk; ++j) {+ const float w = x[i*qk + j];+ int q = (int)roundf(w * id) + 1;+ if (q < 0) q = 0;+ if (q > 3) q = 3;+ const int byte_index = j / 4;+ const int bit_offset = (j % 4) * 2;+ y[i].qs[byte_index] |= ((uint8_t)q << bit_offset);+ }+ }+}+// reference implementation for deterministic creation of model filesvoid quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) {static const int qk = QK4_0;@@ -398,6 +436,26 @@ void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRI}}+void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ static const int qk = QK2_0;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int j = 0; j < qk; ++j) {+ const int byte_index = j / 4;+ const int bit_offset = (j % 4) * 2;+ const uint8_t q = (x[i].qs[byte_index] >> bit_offset) & 0x03;+ // 00=-1, 01=0, 10=+1, 11=+2+ y[i*qk + j] = ((int)q - 1) * d;+ }+ }+}+void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {static const int qk = QK4_0;@@ -2052,6 +2110,20 @@ size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst,return nrow * row_size;}+size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ if (!quant_weights) {+ quantize_row_q2_0_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q2_0, n_per_row);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q2_0, n_per_row);+ char * qrow = (char *)dst;+ for (int64_t row = 0; row < nrow; ++row) {+ quantize_row_q2_0_ref(src, (block_q2_0*)qrow, n_per_row);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {if (!quant_weights) {@@ -5461,6 +5533,10 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte{VALIDATE_ROW_DATA_D_F16_IMPL(block_q1_0, data, nb);} break;+ case GGML_TYPE_Q2_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q2_0, data, nb);+ } break;case GGML_TYPE_Q4_0:{VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb);diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.hindex d56c86da89..75188f1af1 100644--- a/ggml/src/ggml-quants.h+++ b/ggml/src/ggml-quants.h@@ -15,6 +15,7 @@ extern "C" {// QuantizationGGML_API void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k);GGML_API void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);GGML_API void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);GGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);@@ -43,6 +44,7 @@ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_// DequantizationGGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);@@ -93,6 +95,7 @@ GGML_API size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRGGML_API size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);GGML_API size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);GGML_API size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);GGML_API size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);GGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);diff --git a/ggml/src/ggml.c b/ggml/src/ggml.cindex fc46ab0aa0..3e78d3dc0d 100644--- a/ggml/src/ggml.c+++ b/ggml/src/ggml.c@@ -681,6 +681,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {.to_float = (ggml_to_float_t) dequantize_row_q1_0,.from_float_ref = (ggml_from_float_t) quantize_row_q1_0_ref,},+ [GGML_TYPE_Q2_0] = {+ .type_name = "q2_0",+ .blck_size = QK2_0,+ .type_size = sizeof(block_q2_0),+ .is_quantized = true,+ .to_float = (ggml_to_float_t) dequantize_row_q2_0,+ .from_float_ref = (ggml_from_float_t) quantize_row_q2_0_ref,+ },[GGML_TYPE_Q4_0] = {.type_name = "q4_0",.blck_size = QK4_0,@@ -1417,6 +1425,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {case GGML_FTYPE_MOSTLY_Q4_0: wtype = GGML_TYPE_Q4_0; break;case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break;case GGML_FTYPE_MOSTLY_Q1_0: wtype = GGML_TYPE_Q1_0; break;+ case GGML_FTYPE_MOSTLY_Q2_0: wtype = GGML_TYPE_Q2_0; break;case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break;case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break;@@ -7744,6 +7753,7 @@ size_t ggml_quantize_chunk(switch (type) {case GGML_TYPE_Q1_0: result = quantize_q1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q2_0: result = quantize_q2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;case GGML_TYPE_Q4_0: result = quantize_q4_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;case GGML_TYPE_Q4_1: result = quantize_q4_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;case GGML_TYPE_Q5_0: result = quantize_q5_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex cd4cdef899..869e436acd 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -4533,6 +4533,7 @@ class GGMLQuantizationType(IntEnum):MXFP4 = 39NVFP4 = 40Q1_0 = 41+ Q2_0 = 42class ExpertGatingFuncType(IntEnum):@@ -4588,6 +4589,7 @@ class LlamaFileType(IntEnum):MOSTLY_MXFP4_MOE = 38 # except 1d tensorsMOSTLY_NVFP4 = 39 # except 1d tensorsMOSTLY_Q1_0 = 40 # except 1d tensors+ MOSTLY_Q2_0 = 41 # except 1d tensorsGUESSED = 1024 # not specified in the model file@@ -4713,6 +4715,7 @@ GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = {GGMLQuantizationType.MXFP4: (32, 1 + 16),GGMLQuantizationType.NVFP4: (64, 4 + 32),GGMLQuantizationType.Q1_0: (128, 2 + 16),+ GGMLQuantizationType.Q2_0: (64, 2 + 16),}diff --git a/include/llama.h b/include/llama.hindex b89ab758b2..a311ac2023 100644--- a/include/llama.h+++ b/include/llama.h@@ -155,6 +155,7 @@ extern "C" {LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensorsLLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensorsLLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensorsLLAMA_FTYPE_GUESSED = 1024, // not specified in the model file};diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 55554735da..28f8bb7934 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -37,6 +37,7 @@ const char * llama_ftype_name(llama_ftype ftype) {case LLAMA_FTYPE_MOSTLY_F16: name = LLAMA_FTYPE_PREFIX "F16"; break;case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break;case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break;+ case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break;case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break;case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break;case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break;@@ -767,6 +768,7 @@ llama_model_loader::llama_model_loader(case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break;case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break;+ case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break;default:{LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex 847e79f465..aebbc1ffb6 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -380,6 +380,7 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tensocase GGML_TYPE_IQ3_XXS:case GGML_TYPE_IQ3_S: // types on the right: block size 32case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break;+ case GGML_TYPE_Q2_0:case GGML_TYPE_Q2_K:case GGML_TYPE_Q3_K:case GGML_TYPE_TQ1_0:@@ -480,7 +481,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_typeelse if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {new_type = GGML_TYPE_IQ3_S;}- else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0) {+ else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) {new_type = GGML_TYPE_Q4_K;}}@@ -800,6 +801,7 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) {case LLAMA_FTYPE_MOSTLY_BF16: return GGML_TYPE_BF16;case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32;case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0;+ case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0;case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4;diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cppindex 74978e446a..9510ac14ce 100644--- a/tests/test-quantize-fns.cpp+++ b/tests/test-quantize-fns.cpp@@ -158,6 +158,7 @@ static int test_vec_dot_q(bool verbose) {type == GGML_TYPE_Q1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :type == GGML_TYPE_TQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :type == GGML_TYPE_TQ2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :+ type == GGML_TYPE_Q2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :type == GGML_TYPE_Q2_K ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :type == GGML_TYPE_IQ2_S ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :type == GGML_TYPE_Q3_K ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :@@ -183,7 +184,7 @@ static int test_vec_dot_q(bool verbose) {? MAX_DOT_PRODUCT_ERROR_LOWBIT: type == GGML_TYPE_Q1_0? MAX_DOT_PRODUCT_ERROR_BINARY- : type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0+ : type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0? MAX_DOT_PRODUCT_ERROR_TERNARY: type == GGML_TYPE_NVFP4? MAX_DOT_PRODUCT_ERROR_FP4diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cppindex 840eefc2f5..15ef64c4b0 100644--- a/tools/quantize/quantize.cpp+++ b/tools/quantize/quantize.cpp@@ -33,6 +33,7 @@ struct quant_option {static const std::vector<quant_option> QUANT_OPTIONS = {{ "Q1_0", LLAMA_FTYPE_MOSTLY_Q1_0, " 1.125 bpw quantization", },+ { "Q2_0", LLAMA_FTYPE_MOSTLY_Q2_0, " 2.25 bpw quantization (group 64)", },{ "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0, " 4.34G, +0.4685 ppl @ Llama-3-8B", },{ "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 4.78G, +0.4511 ppl @ Llama-3-8B", },{ "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE", },