Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix auto-scheme/alg-ext multiple devices issue (#1107)

Wenhua Cheng3e26b0d90204
diff --git a/README.md b/README.md
index 153af044b..c2cc6c3e3 100644
--- a/README.md
+++ b/README.md
@@ -216,7 +216,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")
</details>
-### Adaptive Bits/Dtype Usage
+### Adaptive Schemes (Experimental Feature)
AutoScheme provides an automatic algorithm to generate adaptive mixed bits/data-type quantization recipes.
Please refer to the [user guide](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#autoscheme) for more details on AutoScheme.
~~~python
diff --git a/auto_round/alg_ext.abi3.so b/auto_round/alg_ext.abi3.so
index f3e73ba72..699905b5d 100755
Binary files a/auto_round/alg_ext.abi3.so and b/auto_round/alg_ext.abi3.so differ
diff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.so
index 2651f8cf5..e8ed9622a 100644
Binary files a/auto_round/auto_scheme/default_alg.abi3.so and b/auto_round/auto_scheme/default_alg.abi3.so differ
diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py
index bc76a5543..8572a920a 100644
--- a/auto_round/auto_scheme/gen_auto_scheme.py
+++ b/auto_round/auto_scheme/gen_auto_scheme.py
@@ -74,6 +74,7 @@ class GenScheme:
else self.auto_scheme.enable_torch_compile
)
self.disable_opt_rtn = self.auto_scheme.disable_opt_rtn
+ self.min_avg_bit, self.max_avg_bit, self.min_avg_bit_scheme = self.compute_avg_bit_range()
self._check_configs()
def _check_configs(self) -> None:
@@ -84,9 +85,9 @@ class GenScheme:
if not isinstance(self.dataset, str):
raise TypeError("`dataset` must be a string, got {type(self.dataset).__name__}.")
- min_avg_bit, max_avg_bit = self.compute_avg_bit_range()
target = self.auto_scheme.avg_bits
-
+ min_avg_bit = self.min_avg_bit
+ max_avg_bit = self.max_avg_bit
logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target)
if abs(target - min_avg_bit) < 1e-3 or abs(target - max_avg_bit) < 1e-3:
if abs(target - min_avg_bit) < 1e-3:
@@ -119,6 +120,7 @@ class GenScheme:
enable_torch_compile=self.enable_torch_compile,
disable_opt_rtn=self.disable_opt_rtn,
low_gpu_mem_usage=self.auto_scheme.low_gpu_mem_usage,
+ min_avg_bit_scheme=self.min_avg_bit_scheme,
)
layer_config = self.fallback_gguf_layer_config(layer_config)
return layer_config
@@ -168,7 +170,7 @@ class GenScheme:
return layer_config
- def compute_avg_bit_range(self) -> tuple[float, float]:
+ def compute_avg_bit_range(self) -> tuple[float, float, str | QuantizationScheme]:
"""Compute the min and max average bitwidths among candidate quantization options."""
avg_bits = [
compute_avg_bits_for_scheme(
@@ -180,5 +182,6 @@ class GenScheme:
)[0]
for option in self.auto_scheme.options
]
- self.min_avg_bit, self.max_avg_bit = min(avg_bits), max(avg_bits)
- return self.min_avg_bit, self.max_avg_bit
+ min_avg_bit, max_avg_bit = min(avg_bits), max(avg_bits)
+ min_avg_bit_scheme = self.auto_scheme.options[avg_bits.index(min_avg_bit)]
+ return min_avg_bit, max_avg_bit, min_avg_bit_scheme
diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.py
index b355b4703..32626240f 100644
--- a/auto_round/auto_scheme/utils.py
+++ b/auto_round/auto_scheme/utils.py
@@ -106,7 +106,6 @@ def compute_avg_bits_for_scheme(
total_params += module.weight.numel()
layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)
total_quantized_bits += layer_bits
-
avg_bits = float(total_quantized_bits) / total_params
if scheme is not None:
@@ -122,7 +121,6 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boo
model: The model to analyze.
ignore_scale_zp_bits: If True, ignores overhead from scale and zero-points.
if scheme is not None:
- apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme)
"""
total_params = 0
@@ -133,8 +131,6 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boo
continue
if not hasattr(module, "weight"):
continue
- # if isinstance(module,torch.nn.Embedding): # Tricky setting for Embedding
- # continue
total_params += module.weight.numel()
layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)
total_quantized_bits += layer_bits
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 54d656b29..fd872288b 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -352,7 +352,7 @@ class BaseCompressor(object):
self.lr = lr
if self.bits <= 2 and (self.iters < 1000 or not enable_alg_ext):
logger.warning(
- "For bits <= 2, it is recommended to enable `auto-round-best` " "and set `--enable_alg_ext` "
+ "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` "
)
self.minmax_lr = minmax_lr or self.lr
@@ -433,12 +433,19 @@ class BaseCompressor(object):
sys.exit(-1)
all_dtypes = []
+ all_gguf = True
for option in scheme.options:
# Resolve the quantization scheme or data type
dtype = "int"
if isinstance(option, str):
+ if not option.lower().startswith("gguf"):
+ all_gguf = False
+
option = preset_name_to_scheme(option)
+ else:
+ all_gguf = False
+
if isinstance(option, QuantizationScheme):
dtype = option.data_type
elif isinstance(option, dict):
@@ -448,7 +455,7 @@ class BaseCompressor(object):
# Check for mixed data types
unique_dtypes = set(all_dtypes)
- if len(unique_dtypes) > 1:
+ if len(unique_dtypes) > 1 and not all_gguf:
logger.warning(
"Models with mixed data_types "
"cannot yet be exported to real formats except GGUF. "
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 4b599d3a5..3383d6385 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -247,7 +247,7 @@ class WrapperLinear(torch.nn.Module):
tensor_max=self.weight_max,
data_type=self.data_type,
q_scale_thresh=self.q_scale_thresh,
- imatrix=self.orig_layer.imatrix if hasattr(self.orig_layer, "imatrix") else None,
+ imatrix=self.orig_layer.imatrix.to(weight.device) if hasattr(self.orig_layer, "imatrix") else None,
global_scale=getattr(self, "weight_global_scale", None),
**quant_kwargs,
)