PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix issue where predefined ignore layers incorrectly apply to GGUF (#1340)
Wenhua Chengca13362149af
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 701b9ac53..58b2bae3c 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -868,7 +868,7 @@ def _gen_layer_config(best_loss, best_path = choose_bits_per_layer_with_path(total_scores, target_params_cnt)# print(best_loss, best_path) # TODO better loglayer_config = copy.deepcopy(fixed_layer_scheme)- options = copy.deepcopy(auto_scheme.options)+ options = list(copy.deepcopy(auto_scheme.options))# Replace scheme preset names with actual QuantizationScheme objectsfor index in range(len(options)):if isinstance(options[index], str):diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 3232c4470..b1f46cc1e 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -282,6 +282,9 @@ class BaseCompressor(object):self.layer_config = layer_config# should be set after loading model and set layer_config, cause some special scheme need these.+ # Preserve the original, unparsed scheme for later use in auto scheme generation+ # within `configure_layer_config` (which may need the raw value instead of `self.scheme`).+ self.orig_scheme = copy.deepcopy(scheme)self.scheme, self.is_auto_scheme = self._parse_and_set_scheme(scheme, kwargs)gguf_scheme_name = get_gguf_scheme(self.scheme)@@ -316,15 +319,6 @@ class BaseCompressor(object):self.quant_lm_head = kwargs.pop("quant_lm_head", False)self.ignore_layers = kwargs.pop("ignore_layers", "")- predefined_ignore_layers = get_predefined_ignore_layers(self.model)-- if predefined_ignore_layers:- logger.info(f"Using predefined ignore_layers: {predefined_ignore_layers}")- tmp_str = ",".join(predefined_ignore_layers)- if self.ignore_layers == "":- self.ignore_layers = tmp_str- else:- self.ignore_layers += "," + tmp_strself.supported_types = SUPPORTED_LAYER_TYPESself.inner_supported_types = INNER_SUPPORTED_LAYER_TYPESself.scale_dtype = convert_dtype_str2torch(scale_dtype)@@ -472,9 +466,6 @@ class BaseCompressor(object):self._check_configs()torch.set_printoptions(precision=3, sci_mode=True)- if isinstance(scheme, AutoScheme):- self.layer_config = self._gen_auto_scheme(model, scheme, dataset, self.device_map)-if is_hpex_available():logger.info("habana_frameworks is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401@@ -1514,6 +1505,20 @@ class BaseCompressor(object):return inputs, q_inputsdef configure_layer_config(self, enable_gguf_official_mixed: None | bool = True):+ is_gguf_format = (f := getattr(self, "formats", None)) is not None and len(f) > 0 and f[0].is_gguf()+ if not is_gguf_format:+ predefined_ignore_layers = get_predefined_ignore_layers(self.model)+ if predefined_ignore_layers:+ logger.info(f"Using predefined ignore_layers: {predefined_ignore_layers}")+ tmp_str = ",".join(predefined_ignore_layers)+ if self.ignore_layers == "":+ self.ignore_layers = tmp_str+ else:+ self.ignore_layers += "," + tmp_str++ if self.is_auto_scheme:+ self.layer_config = self._gen_auto_scheme(self.model, self.orig_scheme, self.dataset, self.device_map)+fill_default_value = Trueif self.is_auto_scheme:fill_default_value = Falsediff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 17d3c9f34..7801d8009 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -12,13 +12,12 @@# See the License for the specific language governing permissions and# limitations under the License.import copy-import osimport randomimport reimport sysfrom dataclasses import asdict, fieldsfrom enum import Enum-from typing import Any, Callable, Dict, List, Tuple, Union+from typing import Callable, Unionimport torchimport transformers@@ -27,7 +26,7 @@ from torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelTypefrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme-from auto_round.utils import check_to_quantized, copy_python_files_from_model_cache, is_fp8_linear, is_fp8_model+from auto_round.utils import check_to_quantized, is_fp8_linear, is_fp8_modelclass BackendDataType(str, Enum):