PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add sym & asym gguf quant for gguf baseline (iter==0) (#573)
Heng Guofde99cc84f4e
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex a16a730f1..72e72d9c5 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -95,7 +95,7 @@ def quant_tensor_sym_dq(super_group_size=16,super_bits=6,**kwargs):- """Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp community+ """Quantize and de-quantize tensor symmetrically. full range, credict goes to llamacpp communityArgs:tensor: Tensor containing the tensor to be quantized@@ -198,6 +198,148 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}+@register_dtype("gguf_int_asym_dq")+def quant_tensor_gguf_asym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=8,+ super_bits=6,+ **kwargs):+ """Quantize and de-quantize tensor asymmetrically. For Q2_K, Q4_K, Q5_K.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ from auto_round.export.export_to_gguf.utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES+ from auto_round.export.export_to_gguf.quant_gpu import make_qkx2_quants+++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)++ search_kwargs = {+ 2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},+ 4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},+ 5: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": False}+ }+ if bits not in search_kwargs:+ raise KeyError(f"bits={bits} is not supported by gguf_int_asym_dq, please check.")++ maxq = 2 ** bits - 1+ ggml_type = f"q{bits}_k"+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]+ orig_dtype = tensor.dtype++ tensor = tensor.to(torch.float32)+ n_blocks = tensor.nelement() // block_size+ # q2_k (nb, 16, 16) q4_k/q5_k: (nb, 8, 32)+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)++ scale, int_w, wmin_m = make_qkx2_quants(tensor, bits=bits, **search_kwargs[bits])++ ##conduct double quant+ scale, d_scale = double_quant_tensor(scale, super_bits, q_scale_thresh)+ wmin_m, d_wmin_m = double_quant_tensor(wmin_m, super_bits, q_scale_thresh)++ replace_ids = scale != 0+ scale = scale.unsqueeze(-1)+ wmin_m = wmin_m.unsqueeze(-1)+ int_w[replace_ids] = torch.round(+ (tensor[replace_ids] + wmin_m[replace_ids]) / scale[replace_ids]).clip(0, maxq).to(+ torch.uint8)+ qdq_result = (scale*int_w - wmin_m).to(orig_dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ # zp = round_ste(wmin_m / scale) # remove this later+ scale = scale.view(-1, 1)+ wmin_m = wmin_m.view(-1, 1)+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}+++@register_dtype("gguf_int_sym_dq")+def quant_tensor_gguf_sym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=16,+ super_bits=6,+ **kwargs):+ """Quantize and de-quantize tensor asymmetrically. For Q3_K, Q6_K.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ from auto_round.export.export_to_gguf.utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES+ from auto_round.export.export_to_gguf.quant_gpu import make_q3_quants, make_qx_quant++ if bits not in [3, 6]:+ raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ maxq = 2 ** (bits - 1)+ ggml_type = f"q{bits}_k"+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]+ orig_dtype = tensor.dtype++ tensor = tensor.to(torch.float32)+ n_blocks = tensor.nelement() // block_size+ # (nb, 16, 16)+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)++ if bits == 3:+ scale, int_w = make_q3_quants(tensor, bits=3, do_rmse=True)+ else:+ scale, int_w = make_qx_quant(tensor, bits=6, rmse_type=1, qw=None)++ #conduct double quant+ scale, d_scale = double_quant_tensor_sym(scale, super_bits, q_scale_thresh)++ replace_ids = scale != 0+ scale = scale.unsqueeze(-1)+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130+ int_w[replace_ids] = (torch.round(+ tensor[replace_ids] / scale[replace_ids]).clip(-maxq, maxq - 1) + maxq).to(torch.uint8)+ qdq_result = (scale * (int_w - zp)).to(tensor.dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, {"scale": scale, "d_scale": d_scale}, zp+@register_dtype("int_asym")def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, **kwargs):diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/quant_gpu.pyindex 57407e021..d1e773c65 100644--- a/auto_round/export/export_to_gguf/quant_gpu.py+++ b/auto_round/export/export_to_gguf/quant_gpu.py@@ -158,7 +158,7 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):raise NotImplementedError(f"bits = {bits} is not supported")data = data.reshape(data_shape)sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)- av_x = torch.sqrt(sum_x2 / 32)+ av_x = torch.sqrt(sum_x2 / data.shape[-1])weight = torch.abs(data) + av_xgroup_min = torch.min(data, axis=-1, keepdims=True)[0]diff --git a/auto_round/utils.py b/auto_round/utils.pyindex cd63b3831..8ddeb3ce0 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1227,6 +1227,8 @@ def _gguf_args_check(args_or_ar, format_str=None):if k == "data_type":if re.search("q\d_1", format) and len(formats) > 1:v = "int"+ if re.search("q\d_k", format) and args_or_ar.iters == 0:+ v = f"gguf_{v}"if getattr(args_or_ar, k) != v:unsupport_list.append(f"{k}={getattr(args_or_ar, k)}")reset_list.append(f"{k}={v}")diff --git a/test/test_gguf_format.py b/test/test_gguf_format.pyindex d3ab8cb54..fed60da3b 100644--- a/test/test_gguf_format.py+++ b/test/test_gguf_format.py@@ -192,6 +192,48 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)++ def test_gguf_baseline(self):+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=3,+ group_size=16,+ sym=True,+ iters=0,+ data_type="gguf_int_sym_dq",+ super_group_size=16,+ super_bits=6+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ text = "There is a girl who likes adventure,"+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))+ shutil.rmtree("./saved", ignore_errors=True)++ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=5,+ group_size=32,+ sym=True,+ iters=0,+ data_type="int_asym_dq",+ super_group_size=8,+ super_bits=6+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto")+ text = "There is a girl who likes adventure,"+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))+ shutil.rmtree("./saved", ignore_errors=True)+ self.assertTrue(autoround.data_type == "gguf_int_asym_dq")if __name__ == "__main__":unittest.main()