Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

keep the dtype after qdq (#268)

wenhuach213a70be84a645
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 449b9ec04..ecd026435 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -617,7 +617,7 @@ class AutoRound(object):
@torch.no_grad()
def cache_inter_data(self, block_names, nsamples, layer_names=[], last_cache_name=None):
- """Save the inputs of block_name for calibration. For layers, we cache both of inputs and output.
+ """Save the inputs of block_name for calibration.
This method temporarily replaces the forward method of the model to capture
the inputs passing through the specified block. It then calibrates the model
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index 349aed27a..ef3508950 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -49,12 +49,13 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_d
wmax = wmax_tmp
scale = ((wmax - wmin) / maxq).to(scale_dtype)
scale = torch.clamp(scale, min=q_scale_thresh)
- zp = round_ste(-wmin / scale) # pylint: disable=E1130
+ zp = round_ste(-wmin / scale) # pylint: disable=E1130
scale = scale.unsqueeze(dim=-1)
zp = zp.unsqueeze(dim=-1)
int_w = round_ste(weight / scale + v)
q = torch.clamp(int_w + zp, 0, maxq)
- return scale * (q - zp), scale, zp
+ qdq_result = (scale * (q - zp)).to(weight.dtype)
+ return qdq_result, scale, zp
@register_dtype("int_sym")
@@ -101,7 +102,8 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dt
int_w = round_ste(weight / scale + v)
q = torch.clamp(int_w + zp, 0, maxq)
- return scale * (q - zp), scale, zp
+ qdq_result = (scale * (q - zp)).to(weight.dtype)
+ return qdq_result, scale, zp
def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
@@ -141,4 +143,5 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0
zp = zp.unsqueeze(dim=-1)
int_w = weight / scale + v
q = torch.clamp(int_w + zp, 0, maxq)
- return scale * (q - zp), scale, zp
+ qdq_result = (scale * (q - zp)).to(weight.dtype)
+ return qdq_result, scale, zp