PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
keep the dtype after qdq (#268)
wenhuach213a70be84a645
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 449b9ec04..ecd026435 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -617,7 +617,7 @@ class AutoRound(object):@torch.no_grad()def cache_inter_data(self, block_names, nsamples, layer_names=[], last_cache_name=None):- """Save the inputs of block_name for calibration. For layers, we cache both of inputs and output.+ """Save the inputs of block_name for calibration.This method temporarily replaces the forward method of the model to capturethe inputs passing through the specified block. It then calibrates the modeldiff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 349aed27a..ef3508950 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -49,12 +49,13 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dwmax = wmax_tmpscale = ((wmax - wmin) / maxq).to(scale_dtype)scale = torch.clamp(scale, min=q_scale_thresh)- zp = round_ste(-wmin / scale) # pylint: disable=E1130+ zp = round_ste(-wmin / scale) # pylint: disable=E1130scale = scale.unsqueeze(dim=-1)zp = zp.unsqueeze(dim=-1)int_w = round_ste(weight / scale + v)q = torch.clamp(int_w + zp, 0, maxq)- return scale * (q - zp), scale, zp+ qdq_result = (scale * (q - zp)).to(weight.dtype)+ return qdq_result, scale, zp@register_dtype("int_sym")@@ -101,7 +102,8 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtint_w = round_ste(weight / scale + v)q = torch.clamp(int_w + zp, 0, maxq)- return scale * (q - zp), scale, zp+ qdq_result = (scale * (q - zp)).to(weight.dtype)+ return qdq_result, scale, zpdef quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,@@ -141,4 +143,5 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0zp = zp.unsqueeze(dim=-1)int_w = weight / scale + vq = torch.clamp(int_w + zp, 0, maxq)- return scale * (q - zp), scale, zp+ qdq_result = (scale * (q - zp)).to(weight.dtype)+ return qdq_result, scale, zp