Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: fake format has no FP32 tensor restoration (#2392)

Xin He8447db40f4e2
diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.py
index f873411b7..2c87053dd 100644
--- a/auto_round/export/formats/backends/fake.py
+++ b/auto_round/export/formats/backends/fake.py
@@ -14,6 +14,7 @@
import copy
import glob
+import json
import os
from typing import Any, Callable, Union
@@ -22,7 +23,7 @@ import torch
from auto_round.export.formats.base import OutputFormat
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, is_nv_fp
-from auto_round.utils import copy_python_files_from_model_cache, unsupported_meta_device
+from auto_round.utils import unsupported_meta_device
def _serialize_quantization_config_value(value):
@@ -55,6 +56,7 @@ def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:
return
safetensor_files = sorted(glob.glob(os.path.join(output_dir, "*.safetensors")))
+ rewrote_safetensors = False
for file_path in safetensor_files:
try:
from safetensors.torch import load_file as safe_load_file
@@ -64,9 +66,24 @@ def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:
normalized, changed = _normalize_state_dict_keys(state)
if changed:
safe_save_file(normalized, file_path)
+ rewrote_safetensors = True
except Exception as exc:
logger.warning("Failed to normalize safetensors keys for %s: %s", file_path, exc)
+ index_path = os.path.join(output_dir, "model.safetensors.index.json")
+ if rewrote_safetensors and os.path.exists(index_path):
+ try:
+ with open(index_path) as index_file:
+ index = json.load(index_file)
+ weight_map = index.get("weight_map", {})
+ normalized_weight_map, changed = _normalize_state_dict_keys(weight_map)
+ if changed:
+ index["weight_map"] = normalized_weight_map
+ with open(index_path, "w") as index_file:
+ json.dump(index, index_file, indent=2)
+ except Exception as exc:
+ logger.warning("Failed to normalize safetensors index keys for %s: %s", index_path, exc)
+
bin_files = sorted(glob.glob(os.path.join(output_dir, "*.bin")))
for file_path in bin_files:
try:
@@ -159,13 +176,13 @@ class FakeFormat(OutputFormat):
if has_fake_act_quant:
_rewrite_saved_weights_without_orig_layer(output_dir)
+ from auto_round.export.utils import apply_post_save_source_fixes
+
+ apply_post_save_source_fixes(model, output_dir)
+
if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(output_dir)
processor = kwargs.get("processor", None)
if processor is not None:
processor.save_pretrained(output_dir)
- try:
- copy_python_files_from_model_cache(model, output_dir)
- except Exception as e:
- logger.warning("Skipping source model Python file copy due to error: %s", e)
return model
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index dd70f20d5..9037a234b 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -23,6 +23,7 @@ from auto_round.utils import (
copy_missing_tensors_from_source,
copy_python_files_from_model_cache,
logger,
+ restore_fp32_tensors_from_source,
unsupported_meta_device,
)
@@ -318,6 +319,34 @@ def _restore_original_layer_types(save_dir: str, source_dir: str) -> None:
json.dump(saved_config, f, indent=2)
+def apply_post_save_source_fixes(model: nn.Module, save_dir: str) -> None:
+ """Restore checkpoint artifacts that ``save_pretrained`` does not preserve."""
+ source_dir = _resolve_model_source_dir(model)
+ if source_dir is None:
+ return
+
+ try:
+ restore_fp32_tensors_from_source(source_dir=source_dir, target_dir=save_dir)
+ except Exception as e:
+ logger.warning("Skipping restore of FP32 tensors from source checkpoint due to error: %s", e)
+
+ if not envs.AR_DISABLE_COPY_MTP_WEIGHTS:
+ try:
+ copy_missing_tensors_from_source(source_dir=source_dir, target_dir=save_dir)
+ except Exception as e:
+ logger.warning("Skipping copy of missing tensors from source checkpoint due to error: %s", e)
+
+ try:
+ _restore_original_layer_types(save_dir, source_dir)
+ except Exception as e: # pragma: no cover - best-effort, never block export
+ logger.warning("Skipping restore of original layer_types due to error: %s", e)
+
+ try:
+ copy_python_files_from_model_cache(model, save_dir)
+ except Exception as e:
+ logger.warning("Skipping source model Python file copy due to error: %s", e)
+
+
def save_model(
model: nn.Module,
save_dir: str,
@@ -367,18 +396,7 @@ def save_model(
save_file(state_dict, os.path.join(save_dir, "model.safetensors"))
_save_model_configs(model, save_dir)
- source_dir = _resolve_model_source_dir(model)
-
- # Allow disabling copy_missing_tensors_from_source via env var AR_DISABLE_COPY_MTP_WEIGHTS, default enabled
- if not envs.AR_DISABLE_COPY_MTP_WEIGHTS:
- try:
- if source_dir is not None:
- copy_missing_tensors_from_source(
- source_dir=source_dir,
- target_dir=save_dir,
- )
- except Exception as e:
- logger.warning("Skipping copy of missing tensors from source checkpoint due to error: %s", e)
+ apply_post_save_source_fixes(model, save_dir)
config_path = os.path.join(save_dir, "config.json")
if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
@@ -392,27 +410,11 @@ def save_model(
with open(config_path, "w") as file:
json.dump(data, file, indent=2)
- # transformers' PreTrainedConfig normalizes ``layer_types`` on load/save (via
- # ``remap_legacy_layer_types`` and dataclass post-init), so ``model.save_pretrained``
- # can rewrite the strings (e.g. hybrid-attention Qwen models). Restore the original
- # ``layer_types`` from the source checkpoint so the exported config stays faithful.
- if source_dir is not None:
- try:
- _restore_original_layer_types(save_dir, source_dir)
- except Exception as e: # pragma: no cover - best-effort, never block export
- logger.warning("Skipping restore of original layer_types due to error: %s", e)
-
config_file = "quantization_config.json"
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
- try:
- if source_dir is not None:
- copy_python_files_from_model_cache(model, save_dir)
- except Exception as e:
- logger.warning("Skipping source model Python file copy due to error: %s", e)
-
def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
"""
diff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.py
index 67987b294..62fcc1506 100644
--- a/auto_round/utils/__init__.py
+++ b/auto_round/utils/__init__.py
@@ -29,6 +29,14 @@ def copy_missing_tensors_from_source(*args, **kwargs):
return _copy_missing_tensors_from_source(*args, **kwargs)
+def restore_fp32_tensors_from_source(*args, **kwargs):
+ from auto_round.utils.missing_tensors import (
+ restore_fp32_tensors_from_source as _restore_fp32_tensors_from_source,
+ )
+
+ return _restore_fp32_tensors_from_source(*args, **kwargs)
+
+
import transformers
from packaging.version import Version
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.py
index a02a63549..63d06bdc7 100644
--- a/auto_round/utils/missing_tensors.py
+++ b/auto_round/utils/missing_tensors.py
@@ -128,56 +128,25 @@ def _restore_special_fp32_tensors(
)
-def copy_missing_tensors_from_source(
+def _resolve_tensor_shard_maps(
source_dir: str,
target_dir: str,
-) -> None:
- """Copy tensors from the source checkpoint that are absent from the saved output.
-
- Some parameters (e.g., MTP layers) are not loaded by ``transformers`` and
- therefore not written by ``model.save_pretrained``. This function copies
- those tensors from the original checkpoint into the quantized output dir.
-
- Detection works by comparing **parent layer names**
- (``tensor_name.rsplit(".", 1)[0]``) between the source checkpoint and the
- saved output. A source tensor is considered "missing" when:
- 1. its name is **not** already present in the saved output, **and**
- 2. its parent layer name is **not** among the parent layer names of any
- saved tensor (i.e. the layer was entirely skipped by
- ``save_pretrained``).
-
- No model or config loading is required.
-
- FP8 handling:
- Missing weight tensors in FP8 dtype that have a corresponding
- ``weight_scale_inv`` tensor are dequantized to BF16 before saving.
+) -> tuple[dict[str, str], dict[str, str]] | None:
+ """Resolve ``tensor_name -> shard_file`` maps for *source_dir* and *target_dir*.
- WOQ quantization:
- If the saved ``config.json`` contains a ``quantization_config`` with
- ``quant_method == "auto-round"`` and
- ``packing_format == "auto_round:auto_gptq"``, missing 2-D weight tensors
- will be quantized with RTN and packed into qweight/qzeros/scales format.
-
- Args:
- source_dir: Path to the original (pre-quantization) checkpoint directory
- or a HuggingFace repo id.
- target_dir: Directory to which the quantized model was saved. Must
- contain a ``config.json``.
+ Returns ``None`` when either checkpoint cannot be resolved (missing config,
+ unresolvable source repo, or no safetensors present), in which case callers
+ should skip whatever tensor-level processing they were about to do.
"""
-
try:
from safetensors import safe_open
- from safetensors.torch import save_file
except ImportError:
- logger.warning("safetensors not available, skipping copy of missing tensors from source checkpoint")
- return
+ logger.warning("safetensors not available, skipping tensor comparison against source checkpoint")
+ return None
- # ------------------------------------------------------------------ #
- # Resolve source directory #
- # ------------------------------------------------------------------ #
config_path = os.path.join(target_dir, "config.json")
if not os.path.exists(config_path):
- return
+ return None
if not os.path.isdir(source_dir):
try:
@@ -186,14 +155,11 @@ def copy_missing_tensors_from_source(
source_dir = download_hf_model(source_dir)
except Exception as e:
logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")
- return
+ return None
if not source_dir or not os.path.isdir(source_dir):
- return
+ return None
- # ------------------------------------------------------------------ #
- # Build a mapping: tensor_name -> source shard file path #
- # ------------------------------------------------------------------ #
source_index_file = os.path.join(source_dir, "model.safetensors.index.json")
source_single_file = os.path.join(source_dir, "model.safetensors")
@@ -208,11 +174,8 @@ def copy_missing_tensors_from_source(
for key in f.keys():
source_tensor_to_file[key] = source_single_file
else:
- return
+ return None
- # ------------------------------------------------------------------ #
- # Collect tensor names already present in the saved output #
- # ------------------------------------------------------------------ #
saved_tensor_to_file: dict[str, str] = {}
saved_index_file = os.path.join(target_dir, "model.safetensors.index.json")
saved_single_file = os.path.join(target_dir, "model.safetensors")
@@ -228,11 +191,87 @@ def copy_missing_tensors_from_source(
with safe_open(saved_single_file, framework="pt", device="cpu") as f:
saved_tensor_to_file = {tensor_name: saved_single_file for tensor_name in f.keys()}
else:
+ return None
+
+ return source_tensor_to_file, saved_tensor_to_file
+
+
+def restore_fp32_tensors_from_source(
+ source_dir: str,
+ target_dir: str,
+) -> None:
+ """Restore FP32 source tensors that were downcast to FP16/BF16 in the saved output.
+
+ This is independent of MTP/missing-tensor handling: it must run by default
+ whenever a source checkpoint can be resolved, regardless of whether
+ ``AR_DISABLE_COPY_MTP_WEIGHTS`` disables the (separate) missing-tensor copy
+ performed by :func:`copy_missing_tensors_from_source`.
+ """
+ maps = _resolve_tensor_shard_maps(source_dir, target_dir)
+ if maps is None:
+ return
+ source_tensor_to_file, saved_tensor_to_file = maps
+ _restore_special_fp32_tensors(source_tensor_to_file, saved_tensor_to_file)
+
+
+def copy_missing_tensors_from_source(
+ source_dir: str,
+ target_dir: str,
+) -> None:
+ """Copy tensors from the source checkpoint that are absent from the saved output.
+
+ Some parameters (e.g., MTP layers) are not loaded by ``transformers`` and
+ therefore not written by ``model.save_pretrained``. This function copies
+ those tensors from the original checkpoint into the quantized output dir.
+
+ Detection works by comparing **parent layer names**
+ (``tensor_name.rsplit(".", 1)[0]``) between the source checkpoint and the
+ saved output. A source tensor is considered "missing" when:
+ 1. its name is **not** already present in the saved output, **and**
+ 2. its parent layer name is **not** among the parent layer names of any
+ saved tensor (i.e. the layer was entirely skipped by
+ ``save_pretrained``).
+
+ No model or config loading is required.
+
+ FP8 handling:
+ Missing weight tensors in FP8 dtype that have a corresponding
+ ``weight_scale_inv`` tensor are dequantized to BF16 before saving.
+
+ WOQ quantization:
+ If the saved ``config.json`` contains a ``quantization_config`` with
+ ``quant_method == "auto-round"`` and
+ ``packing_format == "auto_round:auto_gptq"``, missing 2-D weight tensors
+ will be quantized with RTN and packed into qweight/qzeros/scales format.
+
+ Args:
+ source_dir: Path to the original (pre-quantization) checkpoint directory
+ or a HuggingFace repo id.
+ target_dir: Directory to which the quantized model was saved. Must
+ contain a ``config.json``.
+ """
+
+ try:
+ from safetensors import safe_open
+ from safetensors.torch import save_file
+ except ImportError:
+ logger.warning("safetensors not available, skipping copy of missing tensors from source checkpoint")
+ return
+
+ maps = _resolve_tensor_shard_maps(source_dir, target_dir)
+ if maps is None:
return
+ source_tensor_to_file, saved_tensor_to_file = maps
+ # Also restore FP32 tensors here so direct callers of this function keep
+ # getting the restore; ``restore_fp32_tensors_from_source`` is the
+ # preferred entry point and is safe to call again (no-op once restored).
_restore_special_fp32_tensors(source_tensor_to_file, saved_tensor_to_file)
saved_tensor_names = set(saved_tensor_to_file)
+ saved_index_file = os.path.join(target_dir, "model.safetensors.index.json")
+ saved_single_file = os.path.join(target_dir, "model.safetensors")
+
# ------------------------------------------------------------------ #
# Identify missing tensors via block-prefix statistics #
# ------------------------------------------------------------------ #
diff --git a/test/unit/common/utils/test_missing_tensors.py b/test/unit/common/utils/test_missing_tensors.py
index f1ede5321..e651f4f00 100644
--- a/test/unit/common/utils/test_missing_tensors.py
+++ b/test/unit/common/utils/test_missing_tensors.py
@@ -25,6 +25,7 @@ from auto_round.utils.missing_tensors import (
_normalize_tensor_name_for_warning,
copy_missing_tensors_from_source,
quantize_weight_rtn,
+ restore_fp32_tensors_from_source,
split_fused_expert_tensors,
)
@@ -858,3 +859,146 @@ class TestCopyMissingTensorsFromSource:
"Fused talker expert keys should not be treated as missing when "
"their per-expert 2D components are already saved"
)
+
+
+# ===========================================================================
+# restore_fp32_tensors_from_source: decoupled from MTP handling
+#
+# Regression coverage for Qwen/Qwen3.5-0.8B-style checkpoints, which keep
+# ``linear_attn.norm.weight`` in FP32 and additional ``mtp.*`` tensors that are
+# absent from the saved output. Both must be handled by default, independent
+# of each other and of ``AR_DISABLE_COPY_MTP_WEIGHTS``.
+# ===========================================================================
+
+
+class TestRestoreFp32DecoupledFromMtp:
+
+ def _write_qwen35_checkpoints(self, src: str, tgt: str) -> tuple[torch.Tensor, torch.Tensor]:
+ os.makedirs(src, exist_ok=True)
+ os.makedirs(tgt, exist_ok=True)
+ norm_weight = torch.tensor([1.0001, -2.0002, 3.0003], dtype=torch.float32)
+ mtp_weight = torch.randn(32, 64)
+ _save_safetensors(
+ {
+ "model.language_model.layers.0.linear_attn.norm.weight": norm_weight,
+ "model.language_model.mtp.0.fc.weight": mtp_weight,
+ },
+ os.path.join(src, "model.safetensors"),
+ )
+ _save_safetensors(
+ {
+ # Downcast during save, as `model.save_pretrained` does under bf16 dtype.
+ "model.language_model.layers.0.linear_attn.norm.weight": norm_weight.to(torch.bfloat16),
+ "model.language_model.embed_tokens.weight": torch.randn(8, 64),
+ },
+ os.path.join(tgt, "model.safetensors"),
+ )
+ _write_config(tgt)
+ return norm_weight, mtp_weight
+
+ def test_restore_fp32_tensors_from_source_runs_standalone(self, tmp_path):
+ """restore_fp32_tensors_from_source must fix FP32 precision loss on its own,
+ with no dependency on the missing-tensor (MTP) copy logic."""
+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")
+ norm_weight, _ = self._write_qwen35_checkpoints(src, tgt)
+
+ restore_fp32_tensors_from_source(src, tgt)
+
+ result = _load_safetensors(os.path.join(tgt, "model.safetensors"))
+ restored = result["model.language_model.layers.0.linear_attn.norm.weight"]
+ assert restored.dtype == torch.float32
+ assert torch.equal(restored, norm_weight)
+ # The MTP-only tensor must remain untouched by the fp32-only restore.
+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))
+
+ def test_fp32_restore_not_gated_by_mtp_disable_flag(self, tmp_path, monkeypatch):
+ """FP32 restoration must run by default even when AR_DISABLE_COPY_MTP_WEIGHTS
+ disables the (separate) missing-tensor copy, mirroring how `save_model` now
+ calls the two features independently."""
+ import auto_round.envs as envs
+ from auto_round.export.utils import save_model
+
+ src = str(tmp_path / "src")
+ os.makedirs(src)
+ norm_weight = torch.tensor([1.0001, -2.0002, 3.0003], dtype=torch.float32)
+ _save_safetensors(
+ {
+ "model.language_model.layers.0.linear_attn.norm.weight": norm_weight,
+ "model.language_model.mtp.0.fc.weight": torch.randn(32, 64),
+ },
+ os.path.join(src, "model.safetensors"),
+ )
+
+ monkeypatch.setattr(envs, "AR_DISABLE_COPY_MTP_WEIGHTS", True)
+
+ class _FakeModel:
+ name_or_path = src
+
+ def save_pretrained(self, save_dir, max_shard_size=None, safe_serialization=None):
+ pass
+
+ with tempfile.TemporaryDirectory() as tmp_save_dir:
+ _save_safetensors(
+ {
+ "model.language_model.layers.0.linear_attn.norm.weight": norm_weight.to(torch.bfloat16),
+ "model.language_model.embed_tokens.weight": torch.randn(8, 64),
+ },
+ os.path.join(tmp_save_dir, "model.safetensors"),
+ )
+ _write_config(tmp_save_dir)
+
+ save_model(_FakeModel(), tmp_save_dir, immediate_saving=True)
+
+ result = _load_safetensors(os.path.join(tmp_save_dir, "model.safetensors"))
+ restored = result["model.language_model.layers.0.linear_attn.norm.weight"]
+ assert restored.dtype == torch.float32
+ assert torch.equal(restored, norm_weight)
+ # MTP copy is disabled, so the mtp tensor must NOT be copied.
+ assert not os.path.exists(os.path.join(tmp_save_dir, "model_extra_tensors.safetensors"))
+
+ def test_mtp_and_fp32_both_restored_when_enabled(self, tmp_path, monkeypatch):
+ """With the MTP-disable flag off (default), both fp32 restore and MTP tensor
+ copy must be applied together via save_model."""
+ import auto_round.envs as envs
+ from auto_round.export.utils import save_model
+
+ src = str(tmp_path / "src")
+ os.makedirs(src)
+ norm_weight = torch.tensor([1.0001, -2.0002, 3.0003], dtype=torch.float32)
+ mtp_weight = torch.randn(32, 64)
+ _save_safetensors(
+ {
+ "model.language_model.layers.0.linear_attn.norm.weight": norm_weight,
+ "model.language_model.mtp.0.fc.weight": mtp_weight,
+ },
+ os.path.join(src, "model.safetensors"),
+ )
+
+ monkeypatch.setattr(envs, "AR_DISABLE_COPY_MTP_WEIGHTS", False)
+
+ class _FakeModel:
+ name_or_path = src
+
+ def save_pretrained(self, save_dir, max_shard_size=None, safe_serialization=None):
+ pass
+
+ with tempfile.TemporaryDirectory() as tmp_save_dir:
+ _save_safetensors(
+ {
+ "model.language_model.layers.0.linear_attn.norm.weight": norm_weight.to(torch.bfloat16),
+ "model.language_model.embed_tokens.weight": torch.randn(8, 64),
+ },
+ os.path.join(tmp_save_dir, "model.safetensors"),
+ )
+ _write_config(tmp_save_dir)
+
+ save_model(_FakeModel(), tmp_save_dir, immediate_saving=True)
+
+ result = _load_safetensors(os.path.join(tmp_save_dir, "model.safetensors"))
+ restored = result["model.language_model.layers.0.linear_attn.norm.weight"]
+ assert restored.dtype == torch.float32
+ assert torch.equal(restored, norm_weight)
+
+ extra = _load_safetensors(os.path.join(tmp_save_dir, "model_extra_tensors.safetensors"))
+ assert "model.language_model.mtp.0.fc.weight" in extra
+ torch.testing.assert_close(extra["model.language_model.mtp.0.fc.weight"], mtp_weight)
diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.py
index 96a285442..c643d93f1 100644
--- a/test/unit/test_cpu/export/test_fake_format.py
+++ b/test/unit/test_cpu/export/test_fake_format.py
@@ -166,6 +166,25 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):
assert not torch.equal(loaded_model.block.linear.qdq_input(roundtrip_activation), roundtrip_activation)
+def test_fake_format_normalizes_sharded_safetensors_index(tmp_path):
+ from safetensors import safe_open
+ from safetensors.torch import save_file
+
+ shard_name = "model-00001-of-00001.safetensors"
+ original_key = "model.layers.0.orig_layer.weight"
+ normalized_key = "model.layers.0.weight"
+ save_file({original_key: torch.ones(2, 2)}, tmp_path / shard_name)
+ with open(tmp_path / "model.safetensors.index.json", "w") as index_file:
+ json.dump({"metadata": {}, "weight_map": {original_key: shard_name}}, index_file)
+
+ _rewrite_saved_weights_without_orig_layer(str(tmp_path))
+
+ with safe_open(tmp_path / shard_name, framework="pt", device="cpu") as shard:
+ assert list(shard.keys()) == [normalized_key]
+ with open(tmp_path / "model.safetensors.index.json") as index_file:
+ assert json.load(index_file)["weight_map"] == {normalized_key: shard_name}
+
+
class _TinyLoadModel(torch.nn.Module):
def __init__(self, quantization_config):
super().__init__()
@@ -323,6 +342,105 @@ def test_fake_format_still_saves_when_env_disabled(tmp_path):
assert os.path.exists(os.path.join(output_dir, "config.json"))
+def test_fake_format_meta_device_applies_post_save_source_fixes(tmp_path, monkeypatch):
+ model = _SaveableModel()
+ output_dir = str(tmp_path / "meta_device_model")
+ fixup_calls = []
+
+ monkeypatch.setattr("auto_round.export.formats.backends.fake.unsupported_meta_device", lambda model: True)
+ monkeypatch.setattr("auto_round.export.utils.save_config_artifact", lambda model, save_dir: None)
+ monkeypatch.setattr(
+ "auto_round.export.utils.apply_post_save_source_fixes",
+ lambda saved_model, save_dir: fixup_calls.append((saved_model, save_dir)),
+ )
+
+ FakeFormat("fake", PRESET_SCHEMES["INT4"], SimpleNamespace(mllm=False)).save_quantized(
+ output_dir=output_dir,
+ model=model,
+ )
+
+ assert fixup_calls == [(model, output_dir)]
+
+
+class _SaveableSafetensorsModel(torch.nn.Module):
+ """Minimal model whose ``save_pretrained`` writes real safetensors, so the
+ fp32-restore / MTP-copy post-processing in ``FakeFormat.save_quantized`` has
+ something to act on."""
+
+ def __init__(self, source_dir: str, norm_weight: torch.Tensor):
+ super().__init__()
+ self.linear = torch.nn.Linear(4, 3)
+ self.config = SimpleNamespace(_name_or_path=source_dir)
+ self.name_or_path = source_dir
+ self._norm_weight = norm_weight
+
+ def save_pretrained(self, output_dir):
+ from safetensors.torch import save_file
+
+ os.makedirs(output_dir, exist_ok=True)
+ # Simulate transformers downcasting the FP32 tensor to BF16 on save.
+ save_file(
+ {
+ "model.language_model.layers.0.linear_attn.norm.weight": self._norm_weight.to(torch.bfloat16),
+ "model.embed_tokens.weight": torch.randn(4, 4),
+ },
+ os.path.join(output_dir, "model.safetensors"),
+ )
+ with open(os.path.join(output_dir, "config.json"), "w") as config_file:
+ json.dump({}, config_file)
+
+
+def test_fake_format_restores_fp32_and_copies_mtp_tensors(tmp_path, monkeypatch):
+ """Qwen/Qwen3.5-0.8B-style checkpoints keep ``linear_attn.norm.weight`` in FP32
+ and MTP tensors that transformers does not load. The fake export path must
+ restore/copy both by default, not just the ``auto_round`` save path."""
+ from safetensors.torch import save_file
+
+ import auto_round.envs as envs
+
+ source_dir = str(tmp_path / "source")
+ os.makedirs(source_dir)
+ norm_weight = torch.tensor([1.0001, -2.0002, 3.0003, 4.0004], dtype=torch.float32)
+ mtp_weight = torch.randn(8, 4)
+ save_file(
+ {
+ "model.language_model.layers.0.linear_attn.norm.weight": norm_weight,
+ "model.language_model.mtp.0.fc.weight": mtp_weight,
+ },
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ monkeypatch.setattr(envs, "AR_DISABLE_COPY_MTP_WEIGHTS", False)
+
+ model = _SaveableSafetensorsModel(source_dir, norm_weight)
+ output_dir = str(tmp_path / "fake_qwen35")
+
+ FakeFormat("fake", PRESET_SCHEMES["INT4"], SimpleNamespace(mllm=False)).save_quantized(
+ output_dir=output_dir,
+ model=model,
+ inplace=False,
+ serialization_dict={
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ "data_type": "int",
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "to_quant_block_names": ["block"],
+ "supported_types": [torch.nn.Linear],
+ },
+ )
+
+ from safetensors import safe_open
+
+ with safe_open(os.path.join(output_dir, "model.safetensors"), framework="pt", device="cpu") as f:
+ restored = f.get_tensor("model.language_model.layers.0.linear_attn.norm.weight")
+ assert restored.dtype == torch.float32
+ assert torch.equal(restored, norm_weight)
+
+ with safe_open(os.path.join(output_dir, "model_extra_tensors.safetensors"), framework="pt", device="cpu") as f:
+ assert torch.equal(f.get_tensor("model.language_model.mtp.0.fc.weight"), mtp_weight)
+
+
def test_fake_backend_accepts_mxfp_roundtrip_config():
layer_backend = get_layer_backend(
"cpu",