LLM inference in C/C++
C++ ggml
gguf-py : add numpy Q1_0 and Q2_0 de/quantization support (#30630)
Sh1m13495c5805ce92
diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.pyindex 80966b6ef1..cc0772913b 100644--- a/gguf-py/gguf/quants.py+++ b/gguf-py/gguf/quants.py@@ -654,6 +654,59 @@ class TQ2_0(__Quant, qtype=GGMLQuantizationType.TQ2_0):return (d * qs.astype(np.float32))+class Q1_0(__Quant, qtype=GGMLQuantizationType.Q1_0):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ # sequential sum like the C reference, np.sum uses pairwise summation and can round differently+ d = np.cumsum(abs(blocks), axis=-1, dtype=np.float32)[..., -1:] / np.float32(cls.block_size)+ qs = np.packbits(blocks >= 0, axis=-1, bitorder="little")++ d = d.astype(np.float16).view(np.uint8)++ return np.concatenate([d, qs], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ d, qs = np.hsplit(blocks, [2])++ d = d.view(np.float16).astype(np.float32)++ qs = np.unpackbits(qs, axis=-1, bitorder="little")++ return np.where(qs != 0, d, -d)+++class Q2_0(__Quant, qtype=GGMLQuantizationType.Q2_0):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d = abs(blocks).max(axis=-1, keepdims=True)+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = (np_roundf(blocks * id) + 1).clip(0, 3).astype(np.uint8)++ qs = qs.reshape((n_blocks, -1, 4)) << np.array([0, 2, 4, 6], dtype=np.uint8)+ qs = qs[..., 0] | qs[..., 1] | qs[..., 2] | qs[..., 3]++ d = d.astype(np.float16).view(np.uint8)++ return np.concatenate([d, qs], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, qs = np.hsplit(blocks, [2])++ d = d.view(np.float16).astype(np.float32)++ qs = qs.reshape((n_blocks, -1, 1)) >> np.array([0, 2, 4, 6], dtype=np.uint8)+ qs = (qs & 0x03).reshape((n_blocks, -1)).astype(np.int8) - np.int8(1)++ return (d * qs.astype(np.float32))++class MXFP4(__Quant, qtype=GGMLQuantizationType.MXFP4):# e2m1 values (doubled)# ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdfdiff --git a/gguf-py/tests/test_quants.py b/gguf-py/tests/test_quants.pyindex 9aa7c4ae2a..0d7729f95c 100755--- a/gguf-py/tests/test_quants.py+++ b/gguf-py/tests/test_quants.py@@ -69,6 +69,7 @@ class GGMLQuants:"tq1_0", "tq2_0","mxfp4","nvfp4",+ "q1_0", "q2_0","iq2_xxs", "iq2_xs", "iq2_s", "iq3_xxs", "iq3_s", "iq1_s", "iq1_m","iq4_nl", "iq4_xs",):