PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Optimize mtp tensor process by compiling layer names once for all (#1554)
Xin He1111bcd48b5b
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 663186f3f..20514fba1 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -80,6 +80,7 @@ from auto_round.utils import (check_to_quantized,clear_memory,compile_func,+ compress_layer_names,convert_dtype_str2torch,convert_module_to_hp_if_necessary,detect_device,@@ -1629,8 +1630,9 @@ class BaseCompressor(object):is_gguf_format = (f := getattr(self, "formats", None)) is not None and len(f) > 0 and f[0].is_gguf()if not is_gguf_format:predefined_ignore_layers = get_predefined_ignore_layers(self.model)+ compressed_predefined_ignore_layers = compress_layer_names(predefined_ignore_layers)if predefined_ignore_layers:- logger.info(f"Using predefined ignore_layers: {predefined_ignore_layers}")+ logger.info(f"Using predefined ignore_layers: {compressed_predefined_ignore_layers}")tmp_str = ",".join(predefined_ignore_layers)if self.ignore_layers == "":self.ignore_layers = tmp_str@@ -1884,7 +1886,8 @@ class BaseCompressor(object):f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model")if len(unquantized_layers) > 0:- summary_info += f", {unquantized_layers} have not been quantized"+ compressed_unquantized_layers = compress_layer_names(unquantized_layers)+ summary_info += f", {compressed_unquantized_layers} have not been quantized"logger.info(summary_info)self.quantized = Truediff --git a/auto_round/formats.py b/auto_round/formats.pyindex d6f40362a..03213cef6 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -45,6 +45,7 @@ from auto_round.schemes import (from auto_round.utils import (SUPPORTED_FORMATS,check_to_quantized,+ compress_layer_names,copy_python_files_from_model_cache,find_matching_blocks,get_block_names,@@ -149,6 +150,7 @@ def _check_divisible_by_32(ar):default_dict = asdict(preset_name_to_scheme(ar.scheme.upper()))else:default_dict = asdict(ar.scheme)+ skipped_layers = []if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:for n, m in ar.model.named_modules():if type(m) in ar.supported_types or m.__class__.__name__ in ar.inner_supported_types:@@ -159,7 +161,11 @@ def _check_divisible_by_32(ar):continuear.layer_config.setdefault(n, copy.deepcopy(default_dict))ar.layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})- logger.warning_once("some layers are skipped quantization (shape not divisible by 32).")+ skipped_layers.append(n)+ compressed_skipped_layers = compress_layer_names(skipped_layers)+ logger.warning_once(+ f"some layers are skipped quantization (shape not divisible by 32): {compressed_skipped_layers}"+ )class OutputFormat(ABC):diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 8a6c414fe..96b6066e5 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -542,3 +542,51 @@ def parse_layer_config_arg(s: str) -> dict:return resultreturn _dict()+++def compress_layer_names(names: list) -> str:+ """Compress numbered layer names, e.g. layer.0, layer.1, layer.2 → layer.[0-2]."""+ import re as _re++ # group by (prefix, suffix) where the number varies+ from collections import defaultdict++ groups: dict = defaultdict(list)+ singles: list = []+ for name in names:+ m = _re.match(r"^(.*?\.)?(\d+)(\..+)?$", name)+ if m:+ prefix = m.group(1) or ""+ num = int(m.group(2))+ suffix = m.group(3) or ""+ groups[(prefix, suffix)].append(num)+ else:+ singles.append(name)+ parts: list = []+ for (prefix, suffix), nums in groups.items():+ nums_sorted = sorted(set(nums))+ if len(nums_sorted) == 1:+ parts.append(f"{prefix}{nums_sorted[0]}{suffix}")+ else:+ # Build comma-separated contiguous ranges, e.g. [0,2-3,5]+ ranges = []+ start = prev = nums_sorted[0]+ for n in nums_sorted[1:]:+ if n == prev + 1:+ prev = n+ continue+ # Close the current range+ if start == prev:+ ranges.append(str(start))+ else:+ ranges.append(f"{start}-{prev}")+ start = prev = n+ # Close the final range+ if start == prev:+ ranges.append(str(start))+ else:+ ranges.append(f"{start}-{prev}")+ range_str = ",".join(ranges)+ parts.append(f"{prefix}[{range_str}]{suffix}")+ parts.extend(singles)+ return ", ".join(parts)diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 1466acd49..a7f758b3c 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -44,6 +44,7 @@ from typing import Optional, Tupleimport torchfrom auto_round.logger import logger+from auto_round.utils.common import compress_layer_namesfrom auto_round.utils.weight_handler import _dequant_fp8_linear_weight@@ -215,9 +216,17 @@ def copy_missing_tensors_from_source(logger.debug(f"Missing tensors detected: {missing_tensor_names}")+ try:+ from tqdm import tqdm as _tqdm+ except ImportError:+ _tqdm = None++ # Compressed parent-layer summary for display+ parent_summary = compress_layer_names(list({name.rsplit(".", 1)[0] for name in missing_tensor_names}))logger.info(f"Found {len(missing_tensor_names)} tensor(s) in the source checkpoint that are "- f"absent from the saved output (e.g., MTP parameters). Copying them now..."+ f"absent from the saved output (e.g., MTP parameters). Copying them now...\n"+ f" Layers: {parent_summary}")# ------------------------------------------------------------------ #@@ -229,24 +238,51 @@ def copy_missing_tensors_from_source(shard_to_missing.setdefault(shard_file, []).append(tensor_name)missing_tensors_dict: dict = {}- for shard_file, tensor_names in shard_to_missing.items():+ _iter_shards = (+ _tqdm(+ shard_to_missing.items(),+ desc="Loading missing tensors",+ unit="shard",+ total=len(shard_to_missing),+ )+ if _tqdm+ else shard_to_missing.items()+ )+ for shard_file, tensor_names in _iter_shards:with safe_open(shard_file, framework="pt", device="cpu") as f:- for tensor_name in tensor_names:+ _iter_tensors = (+ _tqdm(+ tensor_names,+ desc=f" {os.path.basename(shard_file)}",+ unit="tensor",+ leave=False,+ )+ if _tqdm+ else tensor_names+ )+ for tensor_name in _iter_tensors:missing_tensors_dict[tensor_name] = f.get_tensor(tensor_name)-# ------------------------------------------------------------------ ## FP8 dequantization: if a weight is FP8 and its scale_inv is present ## ------------------------------------------------------------------ #fp8_dtypes = {"torch.float8_e4m3fn", "torch.float8_e5m2", "torch.float8_e4m3fnuz", "torch.float8_e5m2fnuz"}+ # Find FP8 weight candidates upfront to show accurate progress bar+ fp8_candidates = [+ (name, t)+ for name, t in list(missing_tensors_dict.items())+ if str(t.dtype) in fp8_dtypes and name.endswith(".weight")+ ]+dequantized_keys: list = []keys_to_remove: list = []- for tensor_name, tensor in list(missing_tensors_dict.items()):- if str(tensor.dtype) not in fp8_dtypes:- continue- if not tensor_name.endswith(".weight"):- continue+ _fp8_iter = (+ _tqdm(fp8_candidates, desc="Dequantizing FP8 weights", unit="weight")+ if _tqdm and fp8_candidates+ else fp8_candidates+ )+ for tensor_name, tensor in _fp8_iter:# Look for the corresponding scale_inv tensorbase = tensor_name[: -len(".weight")]@@ -280,10 +316,8 @@ def copy_missing_tensors_from_source(missing_tensors_dict.pop(k, None)if dequantized_keys:- logger.info(- f"Dequantized {len(dequantized_keys)} FP8 weight(s) to BF16 before saving: "- f"{dequantized_keys[:5]}{'...' if len(dequantized_keys) > 5 else ''}"- )+ dq_summary = compress_layer_names([k.rsplit(".", 1)[0] for k in dequantized_keys])+ logger.info(f"Dequantized {len(dequantized_keys)} FP8 weight(s) to BF16 before saving: " f"{dq_summary}")if not missing_tensors_dict:return@@ -465,44 +499,70 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -block_name_to_quantize = qconfig.get("block_name_to_quantize", None)extra_config: dict = qconfig.get("extra_config", {}) or {}+ # Pre-compile all valid regex patterns once to avoid repeated re.compile() calls+ # for every tensor lookup (O(N×M) → O(M) compile + O(N×M) match).+ _compiled_patterns: list = []+ for pattern in extra_config:+ try:+ _compiled_patterns.append((_re.compile(pattern), pattern, extra_config[pattern]))+ except _re.error as exc:+ logger.warning(+ "Invalid regex key in extra_config ignored during pre-compilation: %r (%s)",+ pattern,+ exc,+ )++ # Cache resolved layer configs to avoid repeated regex scans for the same name.+ _layer_cfg_cache: dict = {}+def _resolve_layer_cfg(layer_name: str) -> dict:"""Return effective {bits, group_size, sym, data_type} for *layer_name*.Lookup order:1. Exact key match in extra_config.- 2. Among all regex keys that match layer_name, pick the longest pattern- (longer pattern == more specific, e.g. ``.*mtp\\.fc.*`` beats ``.*mtp.*``).+ 2. Among all pre-compiled regex keys that match layer_name, pick the longest+ pattern (longer pattern == more specific).3. Global defaults."""+ if layer_name in _layer_cfg_cache:+ return _layer_cfg_cache[layer_name]override: dict = {}# 1. exact matchif layer_name in extra_config:override = extra_config[layer_name]else:- # 2. collect all matching regex patterns, keep the most specific (longest)+ # 2. use pre-compiled patterns, keep the most specific (longest)best_pattern: str | None = None- for pattern, cfg in extra_config.items():+ for compiled, pattern, cfg in _compiled_patterns:if pattern == layer_name:continue # already handled above- try:- if _re.search(pattern, layer_name):- if best_pattern is None or len(pattern) > len(best_pattern):- best_pattern = pattern- override = cfg- except _re.error:- pass # not a valid regex → skip- return {+ if compiled.search(layer_name):+ if best_pattern is None or len(pattern) > len(best_pattern):+ best_pattern = pattern+ override = cfg+ result = {"bits": override.get("bits", global_bits),"group_size": override.get("group_size", global_group_size),"sym": override.get("sym", global_sym),"data_type": override.get("data_type", "int"),}+ _layer_cfg_cache[layer_name] = result+ return resultdef _is_fp_layer(layer_cfg: dict) -> bool:"""Return True when the resolved config indicates full-precision (no quantization)."""dt = layer_cfg.get("data_type", "int")return layer_cfg["bits"] >= 16 or dt in ("fp", "float", "float16", "bfloat16", "float32")+ def _is_covered_by_extra_config(layer_name: str) -> bool:+ """Return True if layer_name is matched by any entry in extra_config (exact or regex)."""+ if layer_name in extra_config:+ return True+ for compiled, pattern, _ in _compiled_patterns:+ if pattern != layer_name and compiled.search(layer_name):+ return True+ return False+# Identify weight tensors eligible for quantization (2D Linear weights)def _is_eligible(k: str) -> bool:if not k.endswith(".weight"):@@ -512,9 +572,7 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -layer_name = k[: -len(".weight")]layer_cfg = _resolve_layer_cfg(layer_name)# If extra_config explicitly covers this layer, trust its decision- if layer_name in extra_config or any(- _re.search(p, layer_name) for p in extra_config if p != layer_name and _is_valid_regex(p)- ):+ if _is_covered_by_extra_config(layer_name):return not _is_fp_layer(layer_cfg)# Fall back to BLOCK_NAME_TO_IGNORE heuristicif any(block in k for block in BLOCK_NAME_TO_IGNORE):@@ -533,17 +591,18 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -# Update quantization config: extra_config for ignored layers + block_name_to_quantize for new blocksif ignored_weight_keys or weight_keys:- def _update_qcfg(qcfg):+ def _update_qcfg(qcfg, show_log=True):if ignored_weight_keys:ec = qcfg.get("extra_config", {})for k in ignored_weight_keys:layer_name = k[: -len(".weight")]ec.setdefault(layer_name, {"bits": 16, "data_type": "fp"})qcfg["extra_config"] = ec- logger.info(- f"Updated extra_config for {len(ignored_weight_keys)} ignored layer(s): "- f"{[k[: -len('.weight')] for k in ignored_weight_keys]}"- )+ if show_log:+ logger.info(+ f"Updated extra_config for {len(ignored_weight_keys)} ignored layer(s): "+ f"{[k[: -len('.weight')] for k in ignored_weight_keys]}"+ )if weight_keys:existing = qcfg.get("block_name_to_quantize") or []if isinstance(existing, str):@@ -562,7 +621,8 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -b for b in merged if not any(b != other and b.startswith(other + ".") for other in merged)]qcfg["block_name_to_quantize"] = merged- logger.info(f"Updated block_name_to_quantize: {merged}")+ if show_log:+ logger.info(f"Updated block_name_to_quantize: {merged}")return qcfgfor cfg_file in ["config.json", "quantization_config.json"]:@@ -574,9 +634,11 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -with open(cfg_path) as f:cfg_data = json.load(f)if cfg_file == "config.json":- cfg_data["quantization_config"] = _update_qcfg(cfg_data.get("quantization_config", {}))+ cfg_data["quantization_config"] = _update_qcfg(+ cfg_data.get("quantization_config", {}), show_log=True+ )else:- cfg_data = _update_qcfg(cfg_data)+ cfg_data = _update_qcfg(cfg_data, show_log=False)with open(cfg_path, "w") as f:json.dump(cfg_data, f, indent=2)except Exception as e:@@ -585,15 +647,23 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -if not weight_keys:return missing_tensors_dict+ try:+ from tqdm import tqdm as _tqdm_woq+ except ImportError:+ _tqdm_woq = None+logger.info(- f"Applying WOQ to "+ f"Applying WOQ[RTN] to "f"{len(weight_keys)} missing Linear weight(s) (per-layer overrides from extra_config applied)...")new_tensors: dict = {}packed_weight_keys: set = set()- for weight_key in weight_keys:+ _woq_iter = (+ _tqdm_woq(weight_keys, desc="WOQ[RTN] quantizing missing weights", unit="weight") if _tqdm_woq else weight_keys+ )+ for weight_key in _woq_iter:weight = missing_tensors_dict[weight_key]out_features, in_features = weight.shape@@ -608,7 +678,8 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -continuelogger.debug(- f"WOQ [{layer_name}]: bits={bits}, group_size={effective_gs}, sym={sym}, " f"shape={list(weight.shape)}"+ f"WOQ[RTN] [{layer_name}]: bits={bits}, group_size={effective_gs}, sym={sym}, "+ f"shape={list(weight.shape)}")base_name = layer_name