PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of q5_k_s w/ imatrix (#620)
Heng Guo3bbddfcc9673
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 6cbb50302..1eab0d739 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -464,7 +464,7 @@ class AutoRound(object):has_besides_gguf = Trueif has_gguf and has_besides_gguf:raise ValueError("gguf format is not compatible with other formats, please choose only one of them")- if has_gguf and self.iters != 0:+ if has_gguf and self.iters != 0 and self.bits >= 4:logger.warning("We recommend setting `iters=0` when exporting to GGUF format,"" as we have optimized the RTN method for this case."diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 589c5cbb1..6cf41da65 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -320,7 +320,7 @@ def quant_tensor_gguf_asym_dq(elif bits == 4 or bits == 5:sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_Kav_x = torch.sqrt(sigma2)- quant_weights = torch.sqrt(av_x + tensor * tensor)+ quant_weights = torch.abs(tensor) + av_xparams = search_kwargs[bits]scale, wmin_m = iterative_wls_quant_search(tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],@@ -339,15 +339,15 @@ def quant_tensor_gguf_asym_dq(4: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},}- if bits == 2:- sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K- imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)- quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)- elif bits == 4 or bits == 5:- sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K++ sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ if imatrix is None:av_x = torch.sqrt(sigma2)+ quant_weights = torch.abs(av_x + tensor * tensor)+ else:imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)- quant_weights = imatrix * torch.sqrt(av_x + tensor * tensor)+ quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)+weights = imatrix.reshape(1, -1)weights = weights.expand(tensor.numel() // weights.numel(), -1)quant_weights = weights.reshape(tensor.shape)diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 9d1013a10..f45244e67 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -415,10 +415,7 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- tmp_inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)- all_L = torch.clip(- torch.round((blocks + mins.reshape(*mins.shape, 1)) * tmp_inverse_scale.reshape(*scales.shape, 1)), 0,- 3).to(torch.uint8)+ all_L = torch.zeros_like(blocks, dtype=torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]@@ -530,10 +527,11 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)- all_L = torch.round(- (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(- 0,15).to(torch.uint8)+ all_L = torch.zeros_like(blocks, dtype=torch.uint8)+ # inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)+ # all_L = torch.round(+ # (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(+ # 0,15).to(torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]@@ -592,10 +590,7 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)- all_L = torch.round(- (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0,31).to(- torch.uint8)+ all_L = torch.zeros_like(blocks, dtype=torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]@@ -603,8 +598,8 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)- q_scales = torch.round(inv_scales * scales).to(torch.uint8).clip(0, 63)- q_mins = torch.round(inv_mins * mins).to(torch.uint8).clip(0, 63)+ q_scales = torch.round(inv_scales * scales).clip(0, 63)+ q_mins = torch.round(inv_mins * mins).clip(0, 63)if d_scale is None:output_d = max_scales / 63@@ -652,8 +647,7 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scalescales = scale.reshape(-1, QK_K // 16)output_d = d_scale.reshape(-1, 1).to(torch.float32)iscales = torch.where(output_d == 0, 0, 1 / output_d)- inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)- all_L = (torch.round(blocks * inverse_scale).clip(-32, 31) + 32).to(torch.uint8)+ all_L = torch.zeros_like(blocks, dtype=torch.uint8) + 32else:scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)imax = abs(scales).argmax(dim=-1, keepdim=True)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 7499b1630..ac1bb5cbe 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -528,9 +528,20 @@ def tune(args):eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")if args.act_bits <= 8 or eval_gguf_model:if eval_gguf_model:- # gguf floder only contains one file+ # for file in os.listdir(eval_folder):+ # gguf_file = file+ gguf_file = None+ gguf_format = None # Initialize gguf_format to None+ # gguf folder only contains one file+ for format in formats:+ if format.startswith("gguf"):+ gguf_format = format.split(":")[-1].upper()+ if gguf_format is None: # Validate gguf_format after the loop+ logger.error("No valid gguf format found in formats. Please check the input.")+ sys.exit(-1)for file in os.listdir(eval_folder):- gguf_file = file+ if gguf_format in file:+ gguf_file = filelogger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")if eval_model_dtype == "float32" or eval_model_dtype == "auto":@@ -538,6 +549,9 @@ def tune(args):"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"" but may affect accuracy.")+ if gguf_file is None:+ logger.error("Cannot find correct gguf file for evaluation, please check.")+ sys.exit(-1)model = AutoModelForCausalLM.from_pretrained(eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)model.eval()diff --git a/auto_round/utils.py b/auto_round/utils.pyindex e458f168c..48ef3d987 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1242,7 +1242,7 @@ def _gguf_args_check(args_or_ar, format_str=None):reset_list.append(f"{k}={v}")setattr(args_or_ar, k, v)if len(unsupport_list) > 0:- logger.warning(+ logger.info(f"format {format} does not support for {', '.join(unsupport_list)},"f" reset to {', '.join(reset_list)}.")# Removed obsolete commented-out block for improved readability and maintainability.