Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

set predefined ignore layers (#1318)

Wenhua Cheng7c397e148f7e
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index c30502f9e..81d74e325 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -62,7 +62,7 @@ from auto_round.schemes import (
preset_name_to_scheme,
)
from auto_round.sign_sgd import SignSGD
-from auto_round.special_model_handler import update_module
+from auto_round.special_model_handler import get_predefined_ignore_layers, update_module
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_DTYPES,
@@ -314,6 +314,15 @@ class BaseCompressor(object):
self.quant_lm_head = kwargs.pop("quant_lm_head", False)
self.ignore_layers = kwargs.pop("ignore_layers", "")
+ predefined_ignore_layers = get_predefined_ignore_layers(self.model)
+
+ if predefined_ignore_layers:
+ logger.info(f"Using predefined ignore_layers: {predefined_ignore_layers}")
+ tmp_str = ",".join(predefined_ignore_layers)
+ if self.ignore_layers == "":
+ self.ignore_layers = tmp_str
+ else:
+ self.ignore_layers += "," + tmp_str
self.supported_types = SUPPORTED_LAYER_TYPES
self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index c7e5b5d90..4c2b4bcbc 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -11,6 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import re
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
import torch
import auto_round.modelling as auto_round_modelling
@@ -124,3 +128,119 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
)
return 1, accumulate_steps
return batch_size, gradient_accumulate_steps
+
+
+class ModelNameMatcher:
+ """model.config.name_or_path"""
+
+ def __init__(self, pattern: str, mode="in"):
+ self.pattern = pattern
+ self.mode = mode
+
+ def __call__(self, model) -> bool:
+ name = getattr(model.config, "name_or_path", "")
+ if self.mode == "full":
+ return name == self.pattern
+ elif self.mode == "in":
+ return self.pattern in name
+ elif self.mode == "regex":
+ return re.search(self.pattern, name) is not None
+ else:
+ raise ValueError("unsupported mode {self.mode}")
+
+ """Matches config.architectures."""
+
+
+class ArchitectureMatcher:
+ """match config.architectures"""
+
+ def __init__(self, arch: str, mode="in"):
+ self.arch = arch
+ self.mode = mode
+
+ def __call__(self, model) -> bool:
+ archs = getattr(model.config, "architectures", [])
+ archs_str = ",".join(archs) if isinstance(archs, list) else str(archs)
+
+ if self.mode == "full":
+ return archs_str == self.arch
+ elif self.mode == "in":
+ return self.arch in archs_str
+ elif self.mode == "regex":
+ return re.search(self.arch, archs_str) is not None
+ else:
+ raise ValueError(f"unsupported mode {self.mode}")
+
+
+@dataclass
+class PreDefinedIgnoreLayers:
+ matchers: list[Callable[[Any], bool]]
+ ignore_layers: list[str] = field(default_factory=list)
+
+
+_PRE_DEFINED_IGNORE_LAYERS: list[PreDefinedIgnoreLayers] = []
+
+
+def register_ignore_layers(
+ matchers: list[Callable[[Any], bool]], ignore_layers: list[str | Callable[[torch.nn.Module], str | list[str]]]
+):
+ rule = PreDefinedIgnoreLayers(matchers, ignore_layers)
+ _PRE_DEFINED_IGNORE_LAYERS.append(rule)
+
+
+# Qwen3MOE
+register_ignore_layers(
+ matchers=[
+ ArchitectureMatcher(r"Qwen3.*Moe", mode="regex"),
+ ],
+ ignore_layers=[
+ "mlp.gate", # vllm inference issue
+ ],
+)
+
+# longcat
+register_ignore_layers(
+ matchers=[
+ ArchitectureMatcher(r"Longcat", mode="in"),
+ ],
+ ignore_layers=[
+ "classifier", # transforms directly call the weights of this layer
+ ],
+)
+
+
+def get_glm_flash_ignore_layers(model) -> list[str]:
+ num_dense_layer = 1
+ if hasattr(model, "config") and hasattr(model.config, "first_k_dense_replace"):
+ num_dense_layer = model.config.first_k_dense_replace
+ ignore_layers = [f"layers.{i}.mlp" for i in range(num_dense_layer)]
+ return ignore_layers
+
+
+# glmflash
+register_ignore_layers(
+ matchers=[
+ ArchitectureMatcher(r"Glm4MoeLite", mode="in"),
+ ],
+ ignore_layers=[
+ get_glm_flash_ignore_layers, # vllm issue
+ ],
+)
+
+
+def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
+ layers = []
+ for rule in _PRE_DEFINED_IGNORE_LAYERS:
+ if all(m(model) for m in rule.matchers):
+ for ignore_layer in rule.ignore_layers:
+ if isinstance(ignore_layer, str):
+ layers.append(ignore_layer)
+ else:
+ res = ignore_layer(model)
+ if isinstance(res, str):
+ layers.append(res)
+ else:
+ layers.extend(res)
+ break
+
+ return list(dict.fromkeys(layers))