PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix auto-scheme/alg-ext multiple devices issue (#1107)
Wenhua Cheng3e26b0d90204
diff --git a/README.md b/README.mdindex 153af044b..c2cc6c3e3 100644--- a/README.md+++ b/README.md@@ -216,7 +216,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")</details>-### Adaptive Bits/Dtype Usage+### Adaptive Schemes (Experimental Feature)AutoScheme provides an automatic algorithm to generate adaptive mixed bits/data-type quantization recipes.Please refer to the [user guide](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#autoscheme) for more details on AutoScheme.~~~pythondiff --git a/auto_round/alg_ext.abi3.so b/auto_round/alg_ext.abi3.soindex f3e73ba72..699905b5d 100755Binary files a/auto_round/alg_ext.abi3.so and b/auto_round/alg_ext.abi3.so differdiff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.soindex 2651f8cf5..e8ed9622a 100644Binary files a/auto_round/auto_scheme/default_alg.abi3.so and b/auto_round/auto_scheme/default_alg.abi3.so differdiff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex bc76a5543..8572a920a 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -74,6 +74,7 @@ class GenScheme:else self.auto_scheme.enable_torch_compile)self.disable_opt_rtn = self.auto_scheme.disable_opt_rtn+ self.min_avg_bit, self.max_avg_bit, self.min_avg_bit_scheme = self.compute_avg_bit_range()self._check_configs()def _check_configs(self) -> None:@@ -84,9 +85,9 @@ class GenScheme:if not isinstance(self.dataset, str):raise TypeError("`dataset` must be a string, got {type(self.dataset).__name__}.")- min_avg_bit, max_avg_bit = self.compute_avg_bit_range()target = self.auto_scheme.avg_bits-+ min_avg_bit = self.min_avg_bit+ max_avg_bit = self.max_avg_bitlogger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target)if abs(target - min_avg_bit) < 1e-3 or abs(target - max_avg_bit) < 1e-3:if abs(target - min_avg_bit) < 1e-3:@@ -119,6 +120,7 @@ class GenScheme:enable_torch_compile=self.enable_torch_compile,disable_opt_rtn=self.disable_opt_rtn,low_gpu_mem_usage=self.auto_scheme.low_gpu_mem_usage,+ min_avg_bit_scheme=self.min_avg_bit_scheme,)layer_config = self.fallback_gguf_layer_config(layer_config)return layer_config@@ -168,7 +170,7 @@ class GenScheme:return layer_config- def compute_avg_bit_range(self) -> tuple[float, float]:+ def compute_avg_bit_range(self) -> tuple[float, float, str | QuantizationScheme]:"""Compute the min and max average bitwidths among candidate quantization options."""avg_bits = [compute_avg_bits_for_scheme(@@ -180,5 +182,6 @@ class GenScheme:)[0]for option in self.auto_scheme.options]- self.min_avg_bit, self.max_avg_bit = min(avg_bits), max(avg_bits)- return self.min_avg_bit, self.max_avg_bit+ min_avg_bit, max_avg_bit = min(avg_bits), max(avg_bits)+ min_avg_bit_scheme = self.auto_scheme.options[avg_bits.index(min_avg_bit)]+ return min_avg_bit, max_avg_bit, min_avg_bit_schemediff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex b355b4703..32626240f 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -106,7 +106,6 @@ def compute_avg_bits_for_scheme(total_params += module.weight.numel()layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)total_quantized_bits += layer_bits-avg_bits = float(total_quantized_bits) / total_paramsif scheme is not None:@@ -122,7 +121,6 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boomodel: The model to analyze.ignore_scale_zp_bits: If True, ignores overhead from scale and zero-points.if scheme is not None:- apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme)"""total_params = 0@@ -133,8 +131,6 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boocontinueif not hasattr(module, "weight"):continue- # if isinstance(module,torch.nn.Embedding): # Tricky setting for Embedding- # continuetotal_params += module.weight.numel()layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)total_quantized_bits += layer_bitsdiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 54d656b29..fd872288b 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -352,7 +352,7 @@ class BaseCompressor(object):self.lr = lrif self.bits <= 2 and (self.iters < 1000 or not enable_alg_ext):logger.warning(- "For bits <= 2, it is recommended to enable `auto-round-best` " "and set `--enable_alg_ext` "+ "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` ")self.minmax_lr = minmax_lr or self.lr@@ -433,12 +433,19 @@ class BaseCompressor(object):sys.exit(-1)all_dtypes = []+ all_gguf = Truefor option in scheme.options:# Resolve the quantization scheme or data typedtype = "int"if isinstance(option, str):+ if not option.lower().startswith("gguf"):+ all_gguf = False+option = preset_name_to_scheme(option)+ else:+ all_gguf = False+if isinstance(option, QuantizationScheme):dtype = option.data_typeelif isinstance(option, dict):@@ -448,7 +455,7 @@ class BaseCompressor(object):# Check for mixed data typesunique_dtypes = set(all_dtypes)- if len(unique_dtypes) > 1:+ if len(unique_dtypes) > 1 and not all_gguf:logger.warning("Models with mixed data_types ""cannot yet be exported to real formats except GGUF. "diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 4b599d3a5..3383d6385 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -247,7 +247,7 @@ class WrapperLinear(torch.nn.Module):tensor_max=self.weight_max,data_type=self.data_type,q_scale_thresh=self.q_scale_thresh,- imatrix=self.orig_layer.imatrix if hasattr(self.orig_layer, "imatrix") else None,+ imatrix=self.orig_layer.imatrix.to(weight.device) if hasattr(self.orig_layer, "imatrix") else None,global_scale=getattr(self, "weight_global_scale", None),**quant_kwargs,)