PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix double quant sym bug (#616)
Wenhua Cheng21ecd8b4f445
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c2adb83ae..0341fc2b2 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -533,7 +533,7 @@ class AutoRound(object):if not ("gguf" in format_ or "fake" in format_):only_gguf = Falsebreak- if "fake" == formats:+ if len(formats)==1 and "fake" == formats[0]:only_gguf = Falseif only_gguf:self.scale_dtype = torch.float32diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex b999c81ed..589c5cbb1 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -16,7 +16,6 @@ import torchfrom auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.data_type.register import register_dtype-@register_dtype("int_sym_dq")def quant_tensor_sym_dq(tensor,@@ -64,7 +63,6 @@ def quant_tensor_sym_dq(max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)scale = (max_v / maxq).to(scale_dtype)scale = scale.view(-1, super_group_size)-# conduct double quantscale, d_scale = double_quant_tensor_sym(scale, super_bits)@@ -152,7 +150,7 @@ def double_quant_tensor_sym(tensor, bits):wmax = torch.take_along_dim(tensor, imax, dim=-1)scale = wmax / -maxqinverse_scale = torch.where(scale == 0, 0, 1 / scale)- qdq_tensor = (tensor * inverse_scale).clip(-maxq, maxq - 1) * scale+ qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)),-maxq, maxq - 1) * scalereturn qdq_tensor, scale@@ -311,7 +309,7 @@ def quant_tensor_gguf_asym_dq(maxq = 2 ** bits - 1QK_K = 256quant_weights = None- if imatrix is None or (imatrix is not None and torch.sum(imatrix)==0):+ if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):search_kwargs = {2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},@@ -328,7 +326,7 @@ def quant_tensor_gguf_asym_dq(tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],use_mad=params["use_mad"], weights=quant_weights)- scale= scale.reshape(-1, super_group_size)+ scale = scale.reshape(-1, super_group_size)wmin = wmin_m.reshape(-1, super_group_size)scale, d_scale = double_quant_tensor(scale, super_bits)wmin_m, d_wmin_m = double_quant_tensor(wmin, super_bits)@@ -487,9 +485,10 @@ def quant_tensor_gguf_sym_dq(n_blocks = tensor.nelement() // block_size# (nb, 16, 16)tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)- if imatrix is None or (imatrix is not None and torch.sum(imatrix)==0):+ if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):if bits == 3:- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True)+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug+ ##scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)elif bits == 6:scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)else:@@ -508,11 +507,13 @@ def quant_tensor_gguf_sym_dq(# conduct double quantscale, d_scale = double_quant_tensor_sym(scale, super_bits)- replace_ids = scale != 0scale = scale.unsqueeze(-1)zp = torch.full_like(scale, maxq) # pylint: disable=E1130- int_w[replace_ids] = (torch.round(- tensor[replace_ids] / scale[replace_ids]).clip(-maxq, maxq - 1) + maxq).to(torch.uint8)+ inverse_scale = torch.where(scale == 0, 0, 1.0 / scale)+ int_w = torch.round(tensor * inverse_scale).clip(-maxq, maxq - 1) + maxqqdq_result = (scale * (int_w - zp)).to(orig_dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+return qdq_result, {"scale": scale, "d_scale": d_scale}, zp++diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 2c51f16e6..cf4c24568 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -125,6 +125,7 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,shared_exp = shared_exp.to(torch.float32)tensor = tensor / (2 ** shared_exp)tensor = tensor + v+ tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)tensor = tensor * (2 ** shared_exp)diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex a60541c95..82dcbcf85 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -52,7 +52,7 @@ def get_reciprocal(x):FLOAT4_E2M1_MAX = 6.0FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max-+FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).mindef ref_nvfp4_quant(x, global_scale, block_size=16, v=0):assert global_scale.dtype == torch.float32@@ -61,6 +61,7 @@ def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))scale = float8_e4m3fn_ste(scale).to(torch.float32)+ scale = torch.clamp(scale, min=FLOAT8_E4M3_MIN, max=FLOAT8_E4M3_MAX)output_scale = get_reciprocal(scale * get_reciprocal(global_scale))scaled_x = x.to(torch.float32) * output_scale + vdiff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex a979cf0b8..b0dbcd7b9 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -1159,10 +1159,14 @@ class Model(OriModel):device = "xpu"else:device = "cpu"+ data_torch = data_torch.to(torch.float32)+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zpif data_qtype.name.lower().endswith("_k"):d_scale = module.w_d_scale.to(torch.float32)d_wmin_m = module.w_d_wmin_m.to(torch.float32) if hasattr(module, "w_d_wmin_m") else Nonewmin_m = module.w_wmin_m.to(torch.float32) if hasattr(module, "w_wmin_m") else None+data = ggml_quant_gpu(data_torch,data_qtype.name.lower(),diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 4fde404ad..9d1013a10 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -469,30 +469,33 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleoutput_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)if scale is not None:- scales = scale.reshape(-1, QK_K // 16).to(torch.float32)- output_d = d_scale.reshape(-1, 1).to(torch.float32)- iscales = torch.where(output_d == 0, 0, 1 / output_d)- inverse_scale = torch.where(scales==0, 0, 1.0 / scales)- all_L = torch.round(blocks*inverse_scale.unsqueeze(-1)).clip(-4, 3) + 4- all_L = all_L.to(torch.uint8)- else:- scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=True)- imax = abs(scales).argmax(dim=-1, keepdim=True)- max_scales = torch.take_along_dim(scales, imax, dim=-1)- iscales = torch.where(max_scales != 0, -32 / max_scales, 0)- output_d = torch.where(iscales != 0, 1 / iscales, 0)-- q_scales = torch.round(iscales * scales).clip(-32, 31) + 32- q_scales = q_scales.cpu().numpy().astype(np.uint8)- output_scale[:, :8] = (q_scales[:, :8] & 0xF) | ((q_scales[:, 8:] & 0xF) << 4)- hmask = q_scales >> 4+ qdq_scale = scale.reshape(-1, QK_K // 16).to(torch.float32)+ dq_scale=d_scale.reshape(-1, 1).to(torch.float32)+ inverse_dq_scale = torch.where(dq_scale==0,0,1/dq_scale)+ # output_d = d_scale.reshape(-1, 1).to(torch.float32)+ # iscales = torch.where(output_d == 0, 0, 1 / output_d)+ # inverse_scale = torch.where(scales==0, 0, 1.0 / scales)+ # all_L = torch.round(blocks*inverse_scale.unsqueeze(-1)).clip(-4, 3) + 4+ # all_L = all_L.to(torch.uint8)+ else: ## this is correct+ scales, _ = make_q3_quants(blocks, bits=3, do_rmse=True)+ scales_abs_max = abs(scales).argmax(dim=-1, keepdim=True)+ max_scales_mag = torch.take_along_dim(scales, scales_abs_max, dim=-1)+ inverse_dq_scale = torch.where(max_scales_mag != 0, -32 / max_scales_mag, 0)+ dq_scale = torch.where(inverse_dq_scale != 0, 1 / inverse_dq_scale, 0)+ qscale = torch.round(inverse_dq_scale * scales).clip(-32, 31)+ qdq_scale = dq_scale.to(torch.float32) * qscale+++ # q_scales_offset = torch.round(inverse_dq_scale * scales).clip(-32, 31) + 32+ q_scales_offset = torch.round(qdq_scale*inverse_dq_scale).clip(-32, 31) + 32+ q_scales_offset = q_scales_offset.cpu().numpy().astype(np.uint8)+ output_scale[:, :8] = (q_scales_offset[:, :8] & 0xF) | ((q_scales_offset[:, 8:] & 0xF) << 4)+ hmask = q_scales_offset >> 4output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6- sc = torch.round(iscales * scales).clip(-32, 31)- d = output_d.to(torch.float32) * sc- replace_idx = (d != 0)-- all_L[replace_idx] = (torch.round(blocks[replace_idx] / d[replace_idx].unsqueeze(-1)).clip(-4, 3) + 4).to(+ reverse_qdq_scale = torch.where(qdq_scale == 0, 0, 1 / qdq_scale)+ all_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)all_L = all_L.cpu().numpy()@@ -504,9 +507,10 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleall_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121output_qs = output_qs.reshape(nb, 64).astype(np.uint8)- output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)+ dq_scale = dq_scale.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)# [hmask, qs, scale, d]- return np.concatenate([output_hmask, output_qs, output_scale, output_d], axis=-1)+ return np.concatenate([output_hmask, output_qs, output_scale, dq_scale], axis=-1)+@register_qtype("q4_k")@@ -526,9 +530,10 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- inverse_scale = torch.where(scales==0,0,1.0/scales)- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0, 15).to(- torch.uint8)+ inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)+ all_L = torch.round(+ (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(+ 0,15).to(torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]@@ -557,8 +562,8 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wreplace_ids = d_tmp != 0all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 15).to(- torch.uint8)+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(+ 0, 15).to(torch.uint8)all_L = np.clip(all_L.cpu().numpy(), 0, 15).astype(np.uint8)output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)@@ -588,7 +593,8 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmax_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1))*inverse_scale.reshape(*scales.shape, 1)).clip(0, 31).to(+ all_L = torch.round(+ (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0,31).to(torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)@@ -647,7 +653,7 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleoutput_d = d_scale.reshape(-1, 1).to(torch.float32)iscales = torch.where(output_d == 0, 0, 1 / output_d)inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)- all_L = (torch.round(blocks * inverse_scale).clip(-32, 31)+32).to(torch.uint8)+ all_L = (torch.round(blocks * inverse_scale).clip(-32, 31) + 32).to(torch.uint8)else:scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)imax = abs(scales).argmax(dim=-1, keepdim=True)