Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix issue where predefined ignore layers incorrectly apply to GGUF (#1340)

Wenhua Chengca13362149af
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 701b9ac53..58b2bae3c 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -868,7 +868,7 @@ def _gen_layer_config(
best_loss, best_path = choose_bits_per_layer_with_path(total_scores, target_params_cnt)
# print(best_loss, best_path) # TODO better log
layer_config = copy.deepcopy(fixed_layer_scheme)
- options = copy.deepcopy(auto_scheme.options)
+ options = list(copy.deepcopy(auto_scheme.options))
# Replace scheme preset names with actual QuantizationScheme objects
for index in range(len(options)):
if isinstance(options[index], str):
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 3232c4470..b1f46cc1e 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -282,6 +282,9 @@ class BaseCompressor(object):
self.layer_config = layer_config
# should be set after loading model and set layer_config, cause some special scheme need these.
+ # Preserve the original, unparsed scheme for later use in auto scheme generation
+ # within `configure_layer_config` (which may need the raw value instead of `self.scheme`).
+ self.orig_scheme = copy.deepcopy(scheme)
self.scheme, self.is_auto_scheme = self._parse_and_set_scheme(scheme, kwargs)
gguf_scheme_name = get_gguf_scheme(self.scheme)
@@ -316,15 +319,6 @@ class BaseCompressor(object):
self.quant_lm_head = kwargs.pop("quant_lm_head", False)
self.ignore_layers = kwargs.pop("ignore_layers", "")
- predefined_ignore_layers = get_predefined_ignore_layers(self.model)
-
- if predefined_ignore_layers:
- logger.info(f"Using predefined ignore_layers: {predefined_ignore_layers}")
- tmp_str = ",".join(predefined_ignore_layers)
- if self.ignore_layers == "":
- self.ignore_layers = tmp_str
- else:
- self.ignore_layers += "," + tmp_str
self.supported_types = SUPPORTED_LAYER_TYPES
self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
@@ -472,9 +466,6 @@ class BaseCompressor(object):
self._check_configs()
torch.set_printoptions(precision=3, sci_mode=True)
- if isinstance(scheme, AutoScheme):
- self.layer_config = self._gen_auto_scheme(model, scheme, dataset, self.device_map)
-
if is_hpex_available():
logger.info("habana_frameworks is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
@@ -1514,6 +1505,20 @@ class BaseCompressor(object):
return inputs, q_inputs
def configure_layer_config(self, enable_gguf_official_mixed: None | bool = True):
+ is_gguf_format = (f := getattr(self, "formats", None)) is not None and len(f) > 0 and f[0].is_gguf()
+ if not is_gguf_format:
+ predefined_ignore_layers = get_predefined_ignore_layers(self.model)
+ if predefined_ignore_layers:
+ logger.info(f"Using predefined ignore_layers: {predefined_ignore_layers}")
+ tmp_str = ",".join(predefined_ignore_layers)
+ if self.ignore_layers == "":
+ self.ignore_layers = tmp_str
+ else:
+ self.ignore_layers += "," + tmp_str
+
+ if self.is_auto_scheme:
+ self.layer_config = self._gen_auto_scheme(self.model, self.orig_scheme, self.dataset, self.device_map)
+
fill_default_value = True
if self.is_auto_scheme:
fill_default_value = False
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 17d3c9f34..7801d8009 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -12,13 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
-import os
import random
import re
import sys
from dataclasses import asdict, fields
from enum import Enum
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import Callable, Union
import torch
import transformers
@@ -27,7 +26,7 @@ from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme
-from auto_round.utils import check_to_quantized, copy_python_files_from_model_cache, is_fp8_linear, is_fp8_model
+from auto_round.utils import check_to_quantized, is_fp8_linear, is_fp8_model
class BackendDataType(str, Enum):