Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bug of q5_k_s w/ imatrix (#620)

Heng Guo3bbddfcc9673
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 6cbb50302..1eab0d739 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -464,7 +464,7 @@ class AutoRound(object):
has_besides_gguf = True
if has_gguf and has_besides_gguf:
raise ValueError("gguf format is not compatible with other formats, please choose only one of them")
- if has_gguf and self.iters != 0:
+ if has_gguf and self.iters != 0 and self.bits >= 4:
logger.warning(
"We recommend setting `iters=0` when exporting to GGUF format,"
" as we have optimized the RTN method for this case."
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 589c5cbb1..6cf41da65 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -320,7 +320,7 @@ def quant_tensor_gguf_asym_dq(
elif bits == 4 or bits == 5:
sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_K
av_x = torch.sqrt(sigma2)
- quant_weights = torch.sqrt(av_x + tensor * tensor)
+ quant_weights = torch.abs(tensor) + av_x
params = search_kwargs[bits]
scale, wmin_m = iterative_wls_quant_search(
tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],
@@ -339,15 +339,15 @@ def quant_tensor_gguf_asym_dq(
4: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},
5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},
}
- if bits == 2:
- sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
- imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
- quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
- elif bits == 4 or bits == 5:
- sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
+
+ sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K
+ if imatrix is None:
av_x = torch.sqrt(sigma2)
+ quant_weights = torch.abs(av_x + tensor * tensor)
+ else:
imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)
- quant_weights = imatrix * torch.sqrt(av_x + tensor * tensor)
+ quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)
+
weights = imatrix.reshape(1, -1)
weights = weights.expand(tensor.numel() // weights.numel(), -1)
quant_weights = weights.reshape(tensor.shape)
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index 9d1013a10..f45244e67 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -415,10 +415,7 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
- tmp_inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
- all_L = torch.clip(
- torch.round((blocks + mins.reshape(*mins.shape, 1)) * tmp_inverse_scale.reshape(*scales.shape, 1)), 0,
- 3).to(torch.uint8)
+ all_L = torch.zeros_like(blocks, dtype=torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
@@ -530,10 +527,11 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
- inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
- all_L = torch.round(
- (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(
- 0,15).to(torch.uint8)
+ all_L = torch.zeros_like(blocks, dtype=torch.uint8)
+ # inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
+ # all_L = torch.round(
+ # (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(
+ # 0,15).to(torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
@@ -592,10 +590,7 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
- inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
- all_L = torch.round(
- (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0,31).to(
- torch.uint8)
+ all_L = torch.zeros_like(blocks, dtype=torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
@@ -603,8 +598,8 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)
inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)
- q_scales = torch.round(inv_scales * scales).to(torch.uint8).clip(0, 63)
- q_mins = torch.round(inv_mins * mins).to(torch.uint8).clip(0, 63)
+ q_scales = torch.round(inv_scales * scales).clip(0, 63)
+ q_mins = torch.round(inv_mins * mins).clip(0, 63)
if d_scale is None:
output_d = max_scales / 63
@@ -652,8 +647,7 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
scales = scale.reshape(-1, QK_K // 16)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
iscales = torch.where(output_d == 0, 0, 1 / output_d)
- inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
- all_L = (torch.round(blocks * inverse_scale).clip(-32, 31) + 32).to(torch.uint8)
+ all_L = torch.zeros_like(blocks, dtype=torch.uint8) + 32
else:
scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)
imax = abs(scales).argmax(dim=-1, keepdim=True)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 7499b1630..ac1bb5cbe 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -528,9 +528,20 @@ def tune(args):
eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
if args.act_bits <= 8 or eval_gguf_model:
if eval_gguf_model:
- # gguf floder only contains one file
+ # for file in os.listdir(eval_folder):
+ # gguf_file = file
+ gguf_file = None
+ gguf_format = None # Initialize gguf_format to None
+ # gguf folder only contains one file
+ for format in formats:
+ if format.startswith("gguf"):
+ gguf_format = format.split(":")[-1].upper()
+ if gguf_format is None: # Validate gguf_format after the loop
+ logger.error("No valid gguf format found in formats. Please check the input.")
+ sys.exit(-1)
for file in os.listdir(eval_folder):
- gguf_file = file
+ if gguf_format in file:
+ gguf_file = file
logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")
if eval_model_dtype == "float32" or eval_model_dtype == "auto":
@@ -538,6 +549,9 @@ def tune(args):
"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
" but may affect accuracy."
)
+ if gguf_file is None:
+ logger.error("Cannot find correct gguf file for evaluation, please check.")
+ sys.exit(-1)
model = AutoModelForCausalLM.from_pretrained(
eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)
model.eval()
diff --git a/auto_round/utils.py b/auto_round/utils.py
index e458f168c..48ef3d987 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1242,7 +1242,7 @@ def _gguf_args_check(args_or_ar, format_str=None):
reset_list.append(f"{k}={v}")
setattr(args_or_ar, k, v)
if len(unsupport_list) > 0:
- logger.warning(
+ logger.info(
f"format {format} does not support for {', '.join(unsupport_list)},"
f" reset to {', '.join(reset_list)}.")
# Removed obsolete commented-out block for improved readability and maintainability.