Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add sym & asym gguf quant for gguf baseline (iter==0) (#573)

Heng Guofde99cc84f4e
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index a16a730f1..72e72d9c5 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -95,7 +95,7 @@ def quant_tensor_sym_dq(
super_group_size=16,
super_bits=6,
**kwargs):
- """Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp community
+ """Quantize and de-quantize tensor symmetrically. full range, credict goes to llamacpp community
Args:
tensor: Tensor containing the tensor to be quantized
@@ -198,6 +198,148 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_
return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}
+@register_dtype("gguf_int_asym_dq")
+def quant_tensor_gguf_asym_dq(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ super_group_size=8,
+ super_bits=6,
+ **kwargs):
+ """Quantize and de-quantize tensor asymmetrically. For Q2_K, Q4_K, Q5_K.
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ from auto_round.export.export_to_gguf.utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES
+ from auto_round.export.export_to_gguf.quant_gpu import make_qkx2_quants
+
+
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+
+ search_kwargs = {
+ 2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},
+ 4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},
+ 5: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": False}
+ }
+ if bits not in search_kwargs:
+ raise KeyError(f"bits={bits} is not supported by gguf_int_asym_dq, please check.")
+
+ maxq = 2 ** bits - 1
+ ggml_type = f"q{bits}_k"
+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]
+ orig_dtype = tensor.dtype
+
+ tensor = tensor.to(torch.float32)
+ n_blocks = tensor.nelement() // block_size
+ # q2_k (nb, 16, 16) q4_k/q5_k: (nb, 8, 32)
+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+
+ scale, int_w, wmin_m = make_qkx2_quants(tensor, bits=bits, **search_kwargs[bits])
+
+ ##conduct double quant
+ scale, d_scale = double_quant_tensor(scale, super_bits, q_scale_thresh)
+ wmin_m, d_wmin_m = double_quant_tensor(wmin_m, super_bits, q_scale_thresh)
+
+ replace_ids = scale != 0
+ scale = scale.unsqueeze(-1)
+ wmin_m = wmin_m.unsqueeze(-1)
+ int_w[replace_ids] = torch.round(
+ (tensor[replace_ids] + wmin_m[replace_ids]) / scale[replace_ids]).clip(0, maxq).to(
+ torch.uint8)
+ qdq_result = (scale*int_w - wmin_m).to(orig_dtype)
+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
+ # zp = round_ste(wmin_m / scale) # remove this later
+ scale = scale.view(-1, 1)
+ wmin_m = wmin_m.view(-1, 1)
+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}
+
+
+@register_dtype("gguf_int_sym_dq")
+def quant_tensor_gguf_sym_dq(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ super_group_size=16,
+ super_bits=6,
+ **kwargs):
+ """Quantize and de-quantize tensor asymmetrically. For Q3_K, Q6_K.
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ from auto_round.export.export_to_gguf.utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES
+ from auto_round.export.export_to_gguf.quant_gpu import make_q3_quants, make_qx_quant
+
+ if bits not in [3, 6]:
+ raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")
+
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ maxq = 2 ** (bits - 1)
+ ggml_type = f"q{bits}_k"
+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]
+ orig_dtype = tensor.dtype
+
+ tensor = tensor.to(torch.float32)
+ n_blocks = tensor.nelement() // block_size
+ # (nb, 16, 16)
+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
+
+ if bits == 3:
+ scale, int_w = make_q3_quants(tensor, bits=3, do_rmse=True)
+ else:
+ scale, int_w = make_qx_quant(tensor, bits=6, rmse_type=1, qw=None)
+
+ #conduct double quant
+ scale, d_scale = double_quant_tensor_sym(scale, super_bits, q_scale_thresh)
+
+ replace_ids = scale != 0
+ scale = scale.unsqueeze(-1)
+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130
+ int_w[replace_ids] = (torch.round(
+ tensor[replace_ids] / scale[replace_ids]).clip(-maxq, maxq - 1) + maxq).to(torch.uint8)
+ qdq_result = (scale * (int_w - zp)).to(tensor.dtype)
+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_result, {"scale": scale, "d_scale": d_scale}, zp
+
@register_dtype("int_asym")
def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, **kwargs):
diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/quant_gpu.py
index 57407e021..d1e773c65 100644
--- a/auto_round/export/export_to_gguf/quant_gpu.py
+++ b/auto_round/export/export_to_gguf/quant_gpu.py
@@ -158,7 +158,7 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
raise NotImplementedError(f"bits = {bits} is not supported")
data = data.reshape(data_shape)
sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)
- av_x = torch.sqrt(sum_x2 / 32)
+ av_x = torch.sqrt(sum_x2 / data.shape[-1])
weight = torch.abs(data) + av_x
group_min = torch.min(data, axis=-1, keepdims=True)[0]
diff --git a/auto_round/utils.py b/auto_round/utils.py
index cd63b3831..8ddeb3ce0 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1227,6 +1227,8 @@ def _gguf_args_check(args_or_ar, format_str=None):
if k == "data_type":
if re.search("q\d_1", format) and len(formats) > 1:
v = "int"
+ if re.search("q\d_k", format) and args_or_ar.iters == 0:
+ v = f"gguf_{v}"
if getattr(args_or_ar, k) != v:
unsupport_list.append(f"{k}={getattr(args_or_ar, k)}")
reset_list.append(f"{k}={v}")
diff --git a/test/test_gguf_format.py b/test/test_gguf_format.py
index d3ab8cb54..fed60da3b 100644
--- a/test/test_gguf_format.py
+++ b/test/test_gguf_format.py
@@ -192,6 +192,48 @@ class TestGGUF(unittest.TestCase):
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_gguf_baseline(self):
+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=3,
+ group_size=16,
+ sym=True,
+ iters=0,
+ data_type="gguf_int_sym_dq",
+ super_group_size=16,
+ super_bits=6
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake")
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+ text = "There is a girl who likes adventure,"
+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=5,
+ group_size=32,
+ sym=True,
+ iters=0,
+ data_type="int_asym_dq",
+ super_group_size=8,
+ super_bits=6
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake")
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto")
+ text = "There is a girl who likes adventure,"
+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+ self.assertTrue(autoround.data_type == "gguf_int_asym_dq")
if __name__ == "__main__":
unittest.main()