Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix some issues (#655)

Wenhua Cheng2f22e29aff59
diff --git a/README.md b/README.md
index 63c418fc6..046b23a22 100644
--- a/README.md
+++ b/README.md
@@ -34,12 +34,17 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
## What's New
-* [2025.05] AutoRound now offers experimental support for the widely used **GGUF** format. We currently recommend using
- RTN mode (--iters 0) for all the bits except 3. A more advanced algorithm tailored for some specific configurations is likely
- to be introduced in the upcoming release. Example models are
- available on the Intel Hugging Face space, including
- [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0)
- and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0)
+[//]: # (* [2025.07] AutoRound now offers experimental support for the widely used **GGUF** format. We currently recommend using)
+
+[//]: # ( RTN mode (--iters 0) for all the bits except 3. A more advanced algorithm tailored for some specific configurations is likely)
+
+[//]: # ( to be introduced in the upcoming release. Example models are)
+
+[//]: # ( available on the Intel Hugging Face space, including)
+
+[//]: # ( [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0))
+
+[//]: # ( and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0))
* [2025.05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please refer
to [DeepSeek-R1-0528-int2-mixed-sym-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-sym-inc), [DeepSeek-R1-0528-int4-sym-gptq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-gptq-inc-auto-round)
and [DeepSeek-R1-0528-int4-asym-awq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-awq-inc-auto-round) for
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 51bffe934..b52f7bc0d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1934,6 +1934,16 @@ class AutoRound(object):
minmax_params.append(wrapper_linear.params[key])
else:
round_params.append(wrapper_linear.value)
+ if len(round_params) + len(minmax_params) <= 0:
+ dump_info = (
+ f"quantized {layer_name}"
+ )
+ logger.info(dump_info)
+ with torch.no_grad():
+ unwrapper_layer(self.model, wrapper_linear, layer_name, {})
+ mv_module_from_gpu(layer, self.low_cpu_mem_usage)
+
+
if self.enable_minmax_tuning:
optimizer = self.optimizer(
[{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0
@@ -2123,6 +2133,8 @@ class AutoRound(object):
f"layers in the block"
)
logger.info(dump_info)
+ unwrapper_block(block, {}) ## TODO Quant layer should change
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
return output, output
if self.lr_scheduler is None:
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index c43ce3462..543392c62 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -101,11 +101,14 @@ class WrapperLinear(torch.nn.Module):
weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)
self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)
self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)
- self._init_params("value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning)
+ self._init_params("value", p_dtype, weight_reshape.shape, 0,
+ self.enable_round_tuning and self.orig_layer.bits < 16)
# Min-max scale initialization
shape = get_scale_shape(orig_weight, orig_layer.group_size)
- self._init_params("min_scale", p_dtype, shape, 1.0, self.enable_minmax_tuning)
- self._init_params("max_scale", p_dtype, shape, 1.0, self.enable_minmax_tuning)
+ self._init_params("min_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning
+ and self.orig_layer.bits < 16))
+ self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning
+ and self.orig_layer.bits < 16))
self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits,
orig_layer.sym)
@@ -151,6 +154,8 @@ class WrapperLinear(torch.nn.Module):
Returns:
tuple: Quantized weight, scale, and zero point.
"""
+ if self.orig_layer.bits >= 16:
+ return self.orig_layer.weight, None, None
min_scale.data.clamp_(0, 1.0)
max_scale.data.clamp_(0, 1.0)
weight = self.orig_layer.weight
@@ -254,6 +259,8 @@ class WrapperLinear(torch.nn.Module):
if isinstance(scale, dict):
_set_dict_attr(scale, "scale")
+ elif scale is None:
+ self.orig_layer.scale = None
elif scale.numel() > 1:
self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
else: