PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: restore FP32 tensors saved as FP16/BF16 during tensor copying (#2375)
Xin Hea6c612fa3414
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex bb376c248..5ed142955 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -29,6 +29,64 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIGfrom auto_round.logger import logger+def _normalize_tensor_name_for_warning(name: str, numeric_replacement: str = "<idx>") -> str:+ """Normalize tensor names for warning deduplication.++ Replace standalone numeric path segments (e.g. ``layers.12.experts.3``)+ and bracket indices (e.g. ``layers[12]``) with a fixed placeholder+ (``<idx>`` by default) so warning keys are stable across different+ layer/expert ids.+ """+ parts = name.split(".")+ normalized_parts = []+ for part in parts:+ if part.isdigit():+ normalized_parts.append(numeric_replacement)+ continue+ normalized_parts.append(re.sub(r"\[(\d+)\]", f"[{numeric_replacement}]", part))+ return ".".join(normalized_parts)+++def _compact_name_diffs(a: str, b: str) -> tuple[str, str]:+ """Return the minimal differing fragments of two dotted/comma-separated names.++ Handles comma-separated lists by diffing each corresponding part. For each+ dotted name pair, the common prefix and suffix are stripped and only the+ differing middle is returned. If nothing differs, returns the last segment.+ This keeps log messages short and focused on the changed fields.+ """++ def _single_diff(x: str, y: str) -> tuple[str, str]:+ x_parts = x.split(".")+ y_parts = y.split(".")+ # common prefix+ i = 0+ while i < len(x_parts) and i < len(y_parts) and x_parts[i] == y_parts[i]:+ i += 1+ # common suffix+ j = 0+ while j < len(x_parts) - i and j < len(y_parts) - i and x_parts[-1 - j] == y_parts[-1 - j]:+ j += 1+ x_mid = x_parts[i : len(x_parts) - j] if i < len(x_parts) - j else [x_parts[-1]]+ y_mid = y_parts[i : len(y_parts) - j] if i < len(y_parts) - j else [y_parts[-1]]+ return ".".join(x_mid), ".".join(y_mid)++ if "," in a or "," in b:+ a_list = [p.strip() for p in a.split(",")]+ b_list = [p.strip() for p in b.split(",")]+ max_len = max(len(a_list), len(b_list))+ a_list += [""] * (max_len - len(a_list))+ b_list += [""] * (max_len - len(b_list))+ a_out = []+ b_out = []+ for ax, bx in zip(a_list, b_list):+ ad, bd = _single_diff(ax, bx)+ a_out.append(ad)+ b_out.append(bd)+ return ",".join(a_out), ",".join(b_out)+ return _single_diff(a, b)++def download_audiocaps_csv():"""Download AudioCaps train.csv and return the local cache path.@@ -1374,14 +1432,35 @@ def revert_name_with_weight_transforms(name: str, transforms) -> str:from transformers.core_model_loading import rename_source_keyrenamed_key, _ = rename_source_key(name, renamings, converters, reverse=True)+ if renamed_key != name:+ norm_orig = _normalize_tensor_name_for_warning(name, numeric_replacement="[idx]")+ norm_new = _normalize_tensor_name_for_warning(renamed_key, numeric_replacement="[idx]")+ a_diff, b_diff = _compact_name_diffs(norm_orig, norm_new)+ logger.warning_once(+ "Transformers checkpoint->model renaming detected: '%s' -> '%s'. Using reverted checkpoint name.",+ a_diff,+ b_diff,+ )return renamed_keyexcept Exception: # pragma: no cover - defensive: never break saving on renamereturn namedef revert_checkpoint_conversion_mapping(name: str, key_mapping: dict[str, str]) -> str:+ original_name = nameif "," in name:- return ",".join(revert_checkpoint_conversion_mapping(part, key_mapping) for part in name.split(","))+ # handle comma-separated lists by mapping each part independently+ reverted = ",".join(revert_checkpoint_conversion_mapping(part, key_mapping) for part in name.split(","))+ if reverted != original_name:+ norm_orig = _normalize_tensor_name_for_warning(original_name, numeric_replacement="[idx]")+ norm_rev = _normalize_tensor_name_for_warning(reverted, numeric_replacement="[idx]")+ a_diff, b_diff = _compact_name_diffs(norm_orig, norm_rev)+ logger.warning_once(+ "Transformers checkpoint->model flattened conversion detected: '%s' -> '%s'.",+ a_diff,+ b_diff,+ )+ return revertedfor source_pattern, target_patterns in key_mapping.items():if isinstance(target_patterns, str):@@ -1416,6 +1495,15 @@ def revert_checkpoint_conversion_mapping(name: str, key_mapping: dict[str, str])name, n_replace = re.subn(match_pattern, target_pattern, name)# Early exit of the loopif n_replace > 0:+ if name != original_name:+ norm_orig = _normalize_tensor_name_for_warning(original_name, numeric_replacement="[idx]")+ norm_new = _normalize_tensor_name_for_warning(name, numeric_replacement="[idx]")+ a_diff, b_diff = _compact_name_diffs(norm_orig, norm_new)+ logger.warning_once(+ "Transformers checkpoint->model flattened conversion detected [Recover]: '%s' -> '%s'.",+ a_diff,+ b_diff,+ )return namereturn namediff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 7a679cd30..a02a63549 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -39,11 +39,12 @@import jsonimport os+import tempfileimport torchfrom auto_round.logger import logger-from auto_round.utils.common import apply_checkpoint_conversion_mapping, compress_layer_names+from auto_round.utils.common import compress_layer_namesfrom auto_round.utils.model_free_utils import (_normalize_tensor_name_for_warning,quantize_weight_rtn,@@ -52,41 +53,79 @@ from auto_round.utils.model_free_utils import (from auto_round.utils.weight_handler import _dequant_fp8_linear_weight-def _get_conversion_aliases(name: str, model_type: str | None) -> set[str]:- """Return tensor-name aliases produced by transformers checkpoint conversion.-- Uses ``transformers.conversion_mapping.get_checkpoint_conversion_mapping``- to obtain the rename rules for *model_type* and applies them to *name*.- Returns a set containing the original name plus any converted variants.-- Falls back to ``{name}`` when the mapping is unavailable (older transformers- versions, unknown model_type, etc.).- """- aliases = {name}- if not model_type:- return aliases- try:- from transformers.conversion_mapping import get_checkpoint_conversion_mapping-- mappings = get_checkpoint_conversion_mapping(model_type)- if not mappings:- return aliases- # Build a flat {source_pattern: target_pattern} dict (first match wins)- key_mapping: dict[str, str] = {}- for conversion in mappings:- for src in conversion.source_patterns:- if src not in key_mapping:- key_mapping[src] = (- conversion.target_patterns[0]- if isinstance(conversion.target_patterns, list)- else conversion.target_patterns- )- converted = apply_checkpoint_conversion_mapping(name, key_mapping)- if converted != name:- aliases.add(converted)- except (ImportError, AttributeError, TypeError, ValueError):- pass- return aliases+def _restore_special_fp32_tensors(+ source_tensor_to_file: dict[str, str],+ saved_tensor_to_file: dict[str, str],+) -> None:+ """Restore source FP32 tensors that were saved as FP16 or BF16."""+ from safetensors import safe_open+ from safetensors.torch import save_file++ source_to_saved: dict[str, str] = {}+ for source_name in source_tensor_to_file:+ if source_name in saved_tensor_to_file:+ source_to_saved[source_name] = source_name++ source_names_by_shard: dict[str, list[str]] = {}+ for tensor_name in source_to_saved:+ source_names_by_shard.setdefault(source_tensor_to_file[tensor_name], []).append(tensor_name)++ candidates: list[str] = []+ for source_shard, tensor_names in source_names_by_shard.items():+ with safe_open(source_shard, framework="pt", device="cpu") as source_file:+ for tensor_name in tensor_names:+ if source_file.get_slice(tensor_name).get_dtype() == "F32":+ candidates.append(tensor_name)++ target_names_by_shard: dict[str, list[str]] = {}+ for tensor_name in candidates:+ target_name = source_to_saved[tensor_name]+ target_names_by_shard.setdefault(saved_tensor_to_file[target_name], []).append(tensor_name)++ tensors_to_restore: dict[str, torch.Tensor] = {}+ for target_shard, tensor_names in target_names_by_shard.items():+ with safe_open(target_shard, framework="pt", device="cpu") as target_file:+ for tensor_name in tensor_names:+ target_name = source_to_saved[tensor_name]+ if target_file.get_slice(target_name).get_dtype() in {"F16", "BF16"}:+ source_shard = source_tensor_to_file[tensor_name]+ with safe_open(source_shard, framework="pt", device="cpu") as source_file:+ tensors_to_restore[tensor_name] = source_file.get_tensor(tensor_name)++ restore_names_by_shard: dict[str, list[str]] = {}+ for tensor_name in tensors_to_restore:+ restore_names_by_shard.setdefault(saved_tensor_to_file[source_to_saved[tensor_name]], []).append(tensor_name)++ for target_shard, tensor_names in restore_names_by_shard.items():+ with safe_open(target_shard, framework="pt", device="cpu") as target_file:+ metadata = target_file.metadata()+ shard_tensors = {name: target_file.get_tensor(name) for name in target_file.keys()}+ for tensor_name in tensor_names:+ shard_tensors[source_to_saved[tensor_name]] = tensors_to_restore[tensor_name]++ # Preserve the original shard permission bits so the temporary file's+ # 0600 mode is not inherited by the final replacement target.+ original_mode = os.stat(target_shard).st_mode++ temporary_path = None+ try:+ with tempfile.NamedTemporaryFile(+ dir=os.path.dirname(target_shard), prefix=".restore_fp32_", suffix=".safetensors", delete=False+ ) as temporary_file:+ temporary_path = temporary_file.name+ save_file({name: tensor.contiguous() for name, tensor in shard_tensors.items()}, temporary_path, metadata)+ os.replace(temporary_path, target_shard)+ os.chmod(target_shard, original_mode)+ finally:+ if temporary_path and os.path.exists(temporary_path):+ os.remove(temporary_path)++ if tensors_to_restore:+ tensor_summary = compress_layer_names([name.rsplit(".", 1)[0] for name in tensors_to_restore])+ logger.info(+ f"Restored {len(tensors_to_restore)} tensor(s) from FP16/BF16 to their original FP32 values: "+ f"{tensor_summary}."+ )def copy_missing_tensors_from_source(@@ -140,20 +179,6 @@ def copy_missing_tensors_from_source(if not os.path.exists(config_path):return- # Load model_type from config for checkpoint conversion mapping- model_type: str | None = None- try:- with open(config_path) as f:- _cfg = json.load(f)- model_type = _cfg.get("model_type")- if model_type is None:- # For composite models (e.g. VLMs), try text_config- text_cfg = _cfg.get("text_config")- if isinstance(text_cfg, dict):- model_type = text_cfg.get("model_type")- except (json.JSONDecodeError, OSError):- pass-if not os.path.isdir(source_dir):try:from auto_round.utils.model import download_hf_model@@ -188,20 +213,26 @@ def copy_missing_tensors_from_source(# ------------------------------------------------------------------ ## Collect tensor names already present in the saved output ## ------------------------------------------------------------------ #- saved_tensor_names: set = set()+ saved_tensor_to_file: dict[str, str] = {}saved_index_file = os.path.join(target_dir, "model.safetensors.index.json")saved_single_file = os.path.join(target_dir, "model.safetensors")if os.path.exists(saved_index_file):with open(saved_index_file) as f:saved_idx = json.load(f)- saved_tensor_names = set(saved_idx["weight_map"].keys())+ saved_tensor_to_file = {+ tensor_name: os.path.join(target_dir, shard_file)+ for tensor_name, shard_file in saved_idx["weight_map"].items()+ }elif os.path.exists(saved_single_file):with safe_open(saved_single_file, framework="pt", device="cpu") as f:- saved_tensor_names = set(f.keys())+ saved_tensor_to_file = {tensor_name: saved_single_file for tensor_name in f.keys()}else:return+ _restore_special_fp32_tensors(source_tensor_to_file, saved_tensor_to_file)+ saved_tensor_names = set(saved_tensor_to_file)+# ------------------------------------------------------------------ ## Identify missing tensors via block-prefix statistics ## ------------------------------------------------------------------ #@@ -244,39 +275,20 @@ def copy_missing_tensors_from_source(shortcut_block_prefix: set = {name.split(".", 1)[1] for name in saved_block_prefix if "." in name}saved_block_prefix.update(shortcut_block_prefix)- def _name_aliases(name: str) -> set:- """Return a set of equivalent tensor-name variants to match against saved tensors.-- Handles models where the source and saved prefixes differ, e.g.- google/gemma-3-4b-it: ``language_model.model.*`` ↔ ``model.language_model.*``.- Also applies transformers checkpoint conversion mapping (e.g. Nemotron-H- ``backbone.*`` → ``model.*``).- """- aliases = {name}- if name.startswith("language_model.model."):- aliases.add("model.language_model." + name[len("language_model.model.") :])- elif name.startswith("model.language_model."):- aliases.add("language_model.model." + name[len("model.language_model.") :])- # Apply transformers checkpoint conversion mapping (e.g. Nemotron-H)- if model_type:- aliases |= _get_conversion_aliases(name, model_type)- return aliases-def _is_truly_missing(name: str) -> bool:# Special case: Qwen/Qwen3-0.6B-FP8# lm_head is tied but still in source_dir → not missingif name == "lm_head.weight":return False- aliases = _name_aliases(name)- if aliases & saved_tensor_names:+ if name in saved_tensor_names:return False- parents = {a.rsplit(".", 1)[0] for a in aliases}- if parents & saved_parent_layers:+ parent = name.rsplit(".", 1)[0]+ if parent in saved_parent_layers:return False# For split experts, name is changed but block name is the same.- blocks = {_first_numeric_prefix(a) for a in aliases} - {None}- if blocks:- return not (blocks & saved_block_prefix)+ block = _first_numeric_prefix(name)+ if block is not None:+ return block not in saved_block_prefixreturn Truemissing_tensor_names: list = [name for name in source_tensor_to_file if _is_truly_missing(name)]@@ -465,7 +477,7 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -"""import re as _re- BLOCK_NAME_TO_IGNORE = [".shared_expert_gate.", ".mlp.gate.", ".g_proj.", "mtp.fc."]+ BLOCK_NAME_TO_IGNORE = [".shared_expert_gate.", ".gate.", ".g_proj.", "mtp.fc."]qconfig = _get_woq_config_from_dir(target_dir)if qconfig is None:return missing_tensors_dictdiff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 0c321c34e..c872c9e0e 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -35,7 +35,7 @@ import torchfrom auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.logger import loggerfrom auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme-from auto_round.utils.common import to_standard_regex+from auto_round.utils.common import _normalize_tensor_name_for_warning, to_standard_regexfrom auto_round.utils.device import clear_memory, compile_func_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"@@ -99,22 +99,8 @@ _WARNING_INDEX_PLACEHOLDER = "<idx>"_KEEP_FUSED_EXPERT_MODEL_TYPES: frozenset[str] = frozenset({"inkling_mm_model"})-def _normalize_tensor_name_for_warning(name: str, numeric_replacement: str = _WARNING_INDEX_PLACEHOLDER) -> str:- """Normalize tensor names for warning_once deduplication.-- Replace standalone numeric path segments (e.g. ``layers.12.experts.3``)- and bracket indices (e.g. ``layers[12]``) with a fixed placeholder- (``<idx>`` by default) so warning keys are stable across different- layer/expert ids.- """- parts = name.split(".")- normalized_parts = []- for part in parts:- if part.isdigit():- normalized_parts.append(numeric_replacement)- continue- normalized_parts.append(re.sub(r"\[(\d+)\]", f"[{numeric_replacement}]", part))- return ".".join(normalized_parts)+# _normalize_tensor_name_for_warning moved to auto_round.utils.common to+# avoid circular imports; keep compatibility by importing it from there.def _parse_fused_proj_token(token: str) -> tuple[str, str]:diff --git a/test/unit/common/utils/test_missing_tensors.py b/test/unit/common/utils/test_missing_tensors.pyindex 234ad5053..f1ede5321 100644--- a/test/unit/common/utils/test_missing_tensors.py+++ b/test/unit/common/utils/test_missing_tensors.py@@ -442,29 +442,6 @@ class TestCopyMissingTensorsFromSource:copy_missing_tensors_from_source(src, tgt)assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))- def test_transformers_checkpoint_rename_not_copied(self, tmp_path):- """Nemotron-H: source uses 'backbone.' prefix, saved output uses 'model.' prefix.-- The transformers checkpoint conversion mapping maps 'backbone.' → 'model.',- so source tensors with 'backbone.' prefix should NOT be treated as missing- when the saved output has the corresponding 'model.' tensors.- """- src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")- os.makedirs(src)- os.makedirs(tgt)- _save_safetensors(- {"backbone.layers.0.self_attn.q_proj.weight": torch.randn(32, 64)},- os.path.join(src, "model.safetensors"),- )- _save_safetensors(- {"model.layers.0.self_attn.q_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},- os.path.join(tgt, "model.safetensors"),- )- with open(os.path.join(tgt, "config.json"), "w") as f:- json.dump({"model_type": "nemotron_h"}, f)- copy_missing_tensors_from_source(src, tgt)- assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))-def test_known_block_prefix_not_copied(self, tmp_path):src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")os.makedirs(src)@@ -494,13 +471,76 @@ class TestCopyMissingTensorsFromSource:src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")os.makedirs(src)os.makedirs(tgt)- t = torch.randn(64)+ t = torch.randn(64, dtype=torch.bfloat16)_save_safetensors({"mtp.0.norm.weight": t}, os.path.join(src, "model.safetensors"))_save_safetensors({"mtp.0.norm.weight": t}, os.path.join(tgt, "model.safetensors"))_write_config(tgt)copy_missing_tensors_from_source(src, tgt)assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))+ def test_restores_fp32_tensor_saved_as_fp16(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ original = torch.tensor([1.0001, -2.0002], dtype=torch.float32)+ unchanged = torch.tensor([3.0], dtype=torch.float32)+ _save_safetensors(+ {"model.special.weight": original, "model.unchanged.weight": unchanged},+ os.path.join(src, "model.safetensors"),+ )+ _save_safetensors(+ {+ "model.special.weight": original.to(torch.float16),+ "model.unchanged.weight": torch.tensor([9.0], dtype=torch.float32),+ },+ os.path.join(tgt, "model.safetensors"),+ )+ _write_config(tgt)++ copy_missing_tensors_from_source(src, tgt)++ result = _load_safetensors(os.path.join(tgt, "model.safetensors"))+ assert result["model.special.weight"].dtype == torch.float32+ assert torch.equal(result["model.special.weight"], original)+ assert torch.equal(result["model.unchanged.weight"], torch.tensor([9.0], dtype=torch.float32))+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))++ def test_restores_fp32_tensor_in_indexed_shards(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ original = torch.tensor([1.0001, -2.0002], dtype=torch.float32)+ source_shard = "model-00002-of-00002.safetensors"+ target_shard = "model-00001-of-00002.safetensors"+ _save_safetensors({"model.special.weight": original}, os.path.join(src, source_shard))+ _save_safetensors(+ {"model.special.weight": original.to(torch.bfloat16), "model.other.weight": torch.ones(2)},+ os.path.join(tgt, target_shard),+ )+ with open(os.path.join(src, "model.safetensors.index.json"), "w") as f:+ json.dump({"metadata": {}, "weight_map": {"model.special.weight": source_shard}}, f)+ with open(os.path.join(tgt, "model.safetensors.index.json"), "w") as f:+ json.dump(+ {+ "metadata": {},+ "weight_map": {+ "model.special.weight": target_shard,+ "model.other.weight": target_shard,+ },+ },+ f,+ )+ _write_config(tgt)++ copy_missing_tensors_from_source(src, tgt)++ result = _load_safetensors(os.path.join(tgt, target_shard))+ assert result["model.special.weight"].dtype == torch.float32+ assert torch.equal(result["model.special.weight"], original)+ assert torch.equal(result["model.other.weight"], torch.ones(2))+ with open(os.path.join(tgt, "model.safetensors.index.json")) as f:+ assert json.load(f)["weight_map"]["model.special.weight"] == target_shard+def test_detects_multiple_missing_tensors_from_different_blocks(self, tmp_path):"""All source tensors from blocks absent in the saved output are copied."""src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")@@ -734,31 +774,6 @@ class TestCopyMissingTensorsFromSource:assert weight_map["mtp.0.norm.weight"] == "model_extra_tensors.safetensors"assert weight_map["model.embed_tokens.weight"] == "model.safetensors"- def test_known_block_prefix_not_copied_gemma(self, tmp_path):- src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")- os.makedirs(src)- os.makedirs(tgt)- _save_safetensors(- {- "language_model.model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),- "language_model.model.norm.weight": torch.randn(64),- "language_model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),- "language_model.norm.weight": torch.randn(64),- },- os.path.join(src, "model.safetensors"),- )- _save_safetensors(- {- "language_model.model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),- "model.language_model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),- "model.language_model.norm.weight": torch.randn(64),- },- os.path.join(tgt, "model.safetensors"),- )- _write_config(tgt)- copy_missing_tensors_from_source(src, tgt)- assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))-def test_talker_missing_weight_is_never_woq_quantized(self):"""Talker weights must stay BF16/full precision even in WOQ exports."""with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir: