Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix double quant sym bug (#616)

Wenhua Cheng21ecd8b4f445
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index c2adb83ae..0341fc2b2 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -533,7 +533,7 @@ class AutoRound(object):
if not ("gguf" in format_ or "fake" in format_):
only_gguf = False
break
- if "fake" == formats:
+ if len(formats)==1 and "fake" == formats[0]:
only_gguf = False
if only_gguf:
self.scale_dtype = torch.float32
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index b999c81ed..589c5cbb1 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -16,7 +16,6 @@ import torch
from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.data_type.register import register_dtype
-
@register_dtype("int_sym_dq")
def quant_tensor_sym_dq(
tensor,
@@ -64,7 +63,6 @@ def quant_tensor_sym_dq(
max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)
scale = (max_v / maxq).to(scale_dtype)
scale = scale.view(-1, super_group_size)
-
# conduct double quant
scale, d_scale = double_quant_tensor_sym(scale, super_bits)
@@ -152,7 +150,7 @@ def double_quant_tensor_sym(tensor, bits):
wmax = torch.take_along_dim(tensor, imax, dim=-1)
scale = wmax / -maxq
inverse_scale = torch.where(scale == 0, 0, 1 / scale)
- qdq_tensor = (tensor * inverse_scale).clip(-maxq, maxq - 1) * scale
+ qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)),-maxq, maxq - 1) * scale
return qdq_tensor, scale
@@ -311,7 +309,7 @@ def quant_tensor_gguf_asym_dq(
maxq = 2 ** bits - 1
QK_K = 256
quant_weights = None
- if imatrix is None or (imatrix is not None and torch.sum(imatrix)==0):
+ if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):
search_kwargs = {
2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},
4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},
@@ -328,7 +326,7 @@ def quant_tensor_gguf_asym_dq(
tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],
use_mad=params["use_mad"], weights=quant_weights
)
- scale= scale.reshape(-1, super_group_size)
+ scale = scale.reshape(-1, super_group_size)
wmin = wmin_m.reshape(-1, super_group_size)
scale, d_scale = double_quant_tensor(scale, super_bits)
wmin_m, d_wmin_m = double_quant_tensor(wmin, super_bits)
@@ -487,9 +485,10 @@ def quant_tensor_gguf_sym_dq(
n_blocks = tensor.nelement() // block_size
# (nb, 16, 16)
tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
- if imatrix is None or (imatrix is not None and torch.sum(imatrix)==0):
+ if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):
if bits == 3:
- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True)
+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True) ##TODO this has bug
+ ##scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)
elif bits == 6:
scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)
else:
@@ -508,11 +507,13 @@ def quant_tensor_gguf_sym_dq(
# conduct double quant
scale, d_scale = double_quant_tensor_sym(scale, super_bits)
- replace_ids = scale != 0
scale = scale.unsqueeze(-1)
zp = torch.full_like(scale, maxq) # pylint: disable=E1130
- int_w[replace_ids] = (torch.round(
- tensor[replace_ids] / scale[replace_ids]).clip(-maxq, maxq - 1) + maxq).to(torch.uint8)
+ inverse_scale = torch.where(scale == 0, 0, 1.0 / scale)
+ int_w = torch.round(tensor * inverse_scale).clip(-maxq, maxq - 1) + maxq
qdq_result = (scale * (int_w - zp)).to(orig_dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
+
return qdq_result, {"scale": scale, "d_scale": d_scale}, zp
+
+
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index 2c51f16e6..cf4c24568 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -125,6 +125,7 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,
shared_exp = shared_exp.to(torch.float32)
tensor = tensor / (2 ** shared_exp)
tensor = tensor + v
+ tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)
tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)
tensor = tensor * (2 ** shared_exp)
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index a60541c95..82dcbcf85 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -52,7 +52,7 @@ def get_reciprocal(x):
FLOAT4_E2M1_MAX = 6.0
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
-
+FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min
def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
assert global_scale.dtype == torch.float32
@@ -61,6 +61,7 @@ def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
scale = float8_e4m3fn_ste(scale).to(torch.float32)
+ scale = torch.clamp(scale, min=FLOAT8_E4M3_MIN, max=FLOAT8_E4M3_MAX)
output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
scaled_x = x.to(torch.float32) * output_scale + v
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index a979cf0b8..b0dbcd7b9 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -1159,10 +1159,14 @@ class Model(OriModel):
device = "xpu"
else:
device = "cpu"
+ data_torch = data_torch.to(torch.float32)
+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale
+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp
if data_qtype.name.lower().endswith("_k"):
d_scale = module.w_d_scale.to(torch.float32)
d_wmin_m = module.w_d_wmin_m.to(torch.float32) if hasattr(module, "w_d_wmin_m") else None
wmin_m = module.w_wmin_m.to(torch.float32) if hasattr(module, "w_wmin_m") else None
+
data = ggml_quant_gpu(
data_torch,
data_qtype.name.lower(),
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index 4fde404ad..9d1013a10 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -469,30 +469,33 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
if scale is not None:
- scales = scale.reshape(-1, QK_K // 16).to(torch.float32)
- output_d = d_scale.reshape(-1, 1).to(torch.float32)
- iscales = torch.where(output_d == 0, 0, 1 / output_d)
- inverse_scale = torch.where(scales==0, 0, 1.0 / scales)
- all_L = torch.round(blocks*inverse_scale.unsqueeze(-1)).clip(-4, 3) + 4
- all_L = all_L.to(torch.uint8)
- else:
- scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=True)
- imax = abs(scales).argmax(dim=-1, keepdim=True)
- max_scales = torch.take_along_dim(scales, imax, dim=-1)
- iscales = torch.where(max_scales != 0, -32 / max_scales, 0)
- output_d = torch.where(iscales != 0, 1 / iscales, 0)
-
- q_scales = torch.round(iscales * scales).clip(-32, 31) + 32
- q_scales = q_scales.cpu().numpy().astype(np.uint8)
- output_scale[:, :8] = (q_scales[:, :8] & 0xF) | ((q_scales[:, 8:] & 0xF) << 4)
- hmask = q_scales >> 4
+ qdq_scale = scale.reshape(-1, QK_K // 16).to(torch.float32)
+ dq_scale=d_scale.reshape(-1, 1).to(torch.float32)
+ inverse_dq_scale = torch.where(dq_scale==0,0,1/dq_scale)
+ # output_d = d_scale.reshape(-1, 1).to(torch.float32)
+ # iscales = torch.where(output_d == 0, 0, 1 / output_d)
+ # inverse_scale = torch.where(scales==0, 0, 1.0 / scales)
+ # all_L = torch.round(blocks*inverse_scale.unsqueeze(-1)).clip(-4, 3) + 4
+ # all_L = all_L.to(torch.uint8)
+ else: ## this is correct
+ scales, _ = make_q3_quants(blocks, bits=3, do_rmse=True)
+ scales_abs_max = abs(scales).argmax(dim=-1, keepdim=True)
+ max_scales_mag = torch.take_along_dim(scales, scales_abs_max, dim=-1)
+ inverse_dq_scale = torch.where(max_scales_mag != 0, -32 / max_scales_mag, 0)
+ dq_scale = torch.where(inverse_dq_scale != 0, 1 / inverse_dq_scale, 0)
+ qscale = torch.round(inverse_dq_scale * scales).clip(-32, 31)
+ qdq_scale = dq_scale.to(torch.float32) * qscale
+
+
+ # q_scales_offset = torch.round(inverse_dq_scale * scales).clip(-32, 31) + 32
+ q_scales_offset = torch.round(qdq_scale*inverse_dq_scale).clip(-32, 31) + 32
+ q_scales_offset = q_scales_offset.cpu().numpy().astype(np.uint8)
+ output_scale[:, :8] = (q_scales_offset[:, :8] & 0xF) | ((q_scales_offset[:, 8:] & 0xF) << 4)
+ hmask = q_scales_offset >> 4
output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6
- sc = torch.round(iscales * scales).clip(-32, 31)
- d = output_d.to(torch.float32) * sc
- replace_idx = (d != 0)
-
- all_L[replace_idx] = (torch.round(blocks[replace_idx] / d[replace_idx].unsqueeze(-1)).clip(-4, 3) + 4).to(
+ reverse_qdq_scale = torch.where(qdq_scale == 0, 0, 1 / qdq_scale)
+ all_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(
torch.uint8)
all_L = all_L.cpu().numpy()
@@ -504,9 +507,10 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121
output_qs = output_qs.reshape(nb, 64).astype(np.uint8)
- output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)
+ dq_scale = dq_scale.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)
# [hmask, qs, scale, d]
- return np.concatenate([output_hmask, output_qs, output_scale, output_d], axis=-1)
+ return np.concatenate([output_hmask, output_qs, output_scale, dq_scale], axis=-1)
+
@register_qtype("q4_k")
@@ -526,9 +530,10 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
- inverse_scale = torch.where(scales==0,0,1.0/scales)
- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0, 15).to(
- torch.uint8)
+ inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
+ all_L = torch.round(
+ (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(
+ 0,15).to(torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
@@ -557,8 +562,8 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
replace_ids = d_tmp != 0
all_L[replace_ids] = torch.round(
- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 15).to(
- torch.uint8)
+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(
+ 0, 15).to(torch.uint8)
all_L = np.clip(all_L.cpu().numpy(), 0, 15).astype(np.uint8)
output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
@@ -588,7 +593,8 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1))*inverse_scale.reshape(*scales.shape, 1)).clip(0, 31).to(
+ all_L = torch.round(
+ (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0,31).to(
torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)
@@ -647,7 +653,7 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
output_d = d_scale.reshape(-1, 1).to(torch.float32)
iscales = torch.where(output_d == 0, 0, 1 / output_d)
inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
- all_L = (torch.round(blocks * inverse_scale).clip(-32, 31)+32).to(torch.uint8)
+ all_L = (torch.round(blocks * inverse_scale).clip(-32, 31) + 32).to(torch.uint8)
else:
scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)
imax = abs(scales).argmax(dim=-1, keepdim=True)