PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
set predefined ignore layers (#1318)
Wenhua Cheng7c397e148f7e
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex c30502f9e..81d74e325 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -62,7 +62,7 @@ from auto_round.schemes import (preset_name_to_scheme,)from auto_round.sign_sgd import SignSGD-from auto_round.special_model_handler import update_module+from auto_round.special_model_handler import get_predefined_ignore_layers, update_modulefrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_DTYPES,@@ -314,6 +314,15 @@ class BaseCompressor(object):self.quant_lm_head = kwargs.pop("quant_lm_head", False)self.ignore_layers = kwargs.pop("ignore_layers", "")+ predefined_ignore_layers = get_predefined_ignore_layers(self.model)++ if predefined_ignore_layers:+ logger.info(f"Using predefined ignore_layers: {predefined_ignore_layers}")+ tmp_str = ",".join(predefined_ignore_layers)+ if self.ignore_layers == "":+ self.ignore_layers = tmp_str+ else:+ self.ignore_layers += "," + tmp_strself.supported_types = SUPPORTED_LAYER_TYPESself.inner_supported_types = INNER_SUPPORTED_LAYER_TYPESself.scale_dtype = convert_dtype_str2torch(scale_dtype)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex c7e5b5d90..4c2b4bcbc 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -11,6 +11,10 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+import re+from dataclasses import dataclass, field+from typing import Any, Callable+import torchimport auto_round.modelling as auto_round_modelling@@ -124,3 +128,119 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):)return 1, accumulate_stepsreturn batch_size, gradient_accumulate_steps+++class ModelNameMatcher:+ """model.config.name_or_path"""++ def __init__(self, pattern: str, mode="in"):+ self.pattern = pattern+ self.mode = mode++ def __call__(self, model) -> bool:+ name = getattr(model.config, "name_or_path", "")+ if self.mode == "full":+ return name == self.pattern+ elif self.mode == "in":+ return self.pattern in name+ elif self.mode == "regex":+ return re.search(self.pattern, name) is not None+ else:+ raise ValueError("unsupported mode {self.mode}")++ """Matches config.architectures."""+++class ArchitectureMatcher:+ """match config.architectures"""++ def __init__(self, arch: str, mode="in"):+ self.arch = arch+ self.mode = mode++ def __call__(self, model) -> bool:+ archs = getattr(model.config, "architectures", [])+ archs_str = ",".join(archs) if isinstance(archs, list) else str(archs)++ if self.mode == "full":+ return archs_str == self.arch+ elif self.mode == "in":+ return self.arch in archs_str+ elif self.mode == "regex":+ return re.search(self.arch, archs_str) is not None+ else:+ raise ValueError(f"unsupported mode {self.mode}")+++@dataclass+class PreDefinedIgnoreLayers:+ matchers: list[Callable[[Any], bool]]+ ignore_layers: list[str] = field(default_factory=list)+++_PRE_DEFINED_IGNORE_LAYERS: list[PreDefinedIgnoreLayers] = []+++def register_ignore_layers(+ matchers: list[Callable[[Any], bool]], ignore_layers: list[str | Callable[[torch.nn.Module], str | list[str]]]+):+ rule = PreDefinedIgnoreLayers(matchers, ignore_layers)+ _PRE_DEFINED_IGNORE_LAYERS.append(rule)+++# Qwen3MOE+register_ignore_layers(+ matchers=[+ ArchitectureMatcher(r"Qwen3.*Moe", mode="regex"),+ ],+ ignore_layers=[+ "mlp.gate", # vllm inference issue+ ],+)++# longcat+register_ignore_layers(+ matchers=[+ ArchitectureMatcher(r"Longcat", mode="in"),+ ],+ ignore_layers=[+ "classifier", # transforms directly call the weights of this layer+ ],+)+++def get_glm_flash_ignore_layers(model) -> list[str]:+ num_dense_layer = 1+ if hasattr(model, "config") and hasattr(model.config, "first_k_dense_replace"):+ num_dense_layer = model.config.first_k_dense_replace+ ignore_layers = [f"layers.{i}.mlp" for i in range(num_dense_layer)]+ return ignore_layers+++# glmflash+register_ignore_layers(+ matchers=[+ ArchitectureMatcher(r"Glm4MoeLite", mode="in"),+ ],+ ignore_layers=[+ get_glm_flash_ignore_layers, # vllm issue+ ],+)+++def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:+ layers = []+ for rule in _PRE_DEFINED_IGNORE_LAYERS:+ if all(m(model) for m in rule.matchers):+ for ignore_layer in rule.ignore_layers:+ if isinstance(ignore_layer, str):+ layers.append(ignore_layer)+ else:+ res = ignore_layer(model)+ if isinstance(res, str):+ layers.append(res)+ else:+ layers.extend(res)+ break++ return list(dict.fromkeys(layers))