PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from types import SimpleNamespace
import pytest
import torch
from auto_round.compressors.shard_writer import ShardWriter
from auto_round.context.compress import CompressContext
from auto_round.context.model import ModelContext
class _ToyBlock(torch.nn.Module):
def __init__(self):
super().__init__()
self.linear = torch.nn.Linear(4, 4)
class _DiffusionStyleModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.transformer_blocks = torch.nn.ModuleList([_ToyBlock()])
self.proj_out = torch.nn.Linear(4, 2)
self.config = SimpleNamespace(model_type="toy-diffusion")
class _FormatStub:
def get_backend_name(self):
return "auto_round"
def _make_writer(model, output_dir, monkeypatch):
ShardWriter.reset()
compress_context = SimpleNamespace(formats=[_FormatStub()], output_dir=output_dir)
model_context = SimpleNamespace(is_diffusion=False)
monkeypatch.setattr(CompressContext, "get_context", classmethod(lambda cls: compress_context))
monkeypatch.setattr(ModelContext, "get_context", classmethod(lambda cls: model_context))
return ShardWriter(model, bits=4, max_shard_size="1MB", safe_serialization=False)
def test_finalize_saves_tail_layer_when_tie_word_embeddings_missing(tmp_path, monkeypatch):
model = _DiffusionStyleModel()
writer = _make_writer(model, str(tmp_path), monkeypatch)
assert writer.lm_head_name == "proj_out"
assert not hasattr(model.config, "tie_word_embeddings")
writer.save_module(model.transformer_blocks[0], "transformer_blocks.0")
writer.finalize()
shard_path = os.path.join(tmp_path, "model.bin")
saved_tensors = torch.load(shard_path, map_location="cpu")
assert "transformer_blocks.0.linear.weight" in saved_tensors
assert "proj_out.weight" in saved_tensors, "proj_out must be saved when tie_word_embeddings is absent"
assert "proj_out.bias" in saved_tensors
class _LMStyleModel(torch.nn.Module):
"""Model whose config explicitly sets tie_word_embeddings=True."""
def __init__(self):
super().__init__()
self.transformer_blocks = torch.nn.ModuleList([_ToyBlock()])
self.lm_head = torch.nn.Linear(4, 2, bias=False)
self.config = SimpleNamespace(model_type="toy-lm", tie_word_embeddings=True)
class _ToyExperts(torch.nn.Module):
def __init__(self):
super().__init__()
self.is_transposed = False
class _FusedExpertsModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.block = torch.nn.Module()
self.block.experts = _ToyExperts()
self.talker = torch.nn.Module()
self.talker.experts = _ToyExperts()
self.config = SimpleNamespace(model_type="qwen3_omni_moe")
def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path, monkeypatch):
"""Complementary test: when tie_word_embeddings=True the lm_head should be
skipped (not written to disk) and offloaded to meta."""
model = _LMStyleModel()
writer = _make_writer(model, str(tmp_path), monkeypatch)
assert writer.lm_head_name == "lm_head"
writer.save_module(model.transformer_blocks[0], "transformer_blocks.0")
writer.finalize()
shard_path = os.path.join(tmp_path, "model.bin")
saved_tensors = torch.load(shard_path, map_location="cpu")
assert "transformer_blocks.0.linear.weight" in saved_tensors
assert "lm_head.weight" not in saved_tensors, "lm_head must be skipped when tied"
assert model.lm_head.weight.device.type == "meta"
def test_expand_fused_experts_for_skipped_talker_prefix(tmp_path, monkeypatch):
"""Talker fused 3D weights must be expanded to exact per-expert 2D keys.
Real Qwen3-Omni-MoE exports attach reverse checkpoint conversion mappings for
MoE projections. If we apply that mapping to the fused talker tensor before
expanding it, the fused tensor is saved under a wildcard key such as
``talker.experts.*.gate_proj.weight``. That breaks reload because
transformers expects concrete per-expert 2D keys after save_pretrained.
"""
model = _FusedExpertsModel()
writer = _make_writer(model, str(tmp_path), monkeypatch)
writer.reverse_checkpoint_conversion_mapping = {
r"experts\.gate_up_proj$": ["experts.*.gate_proj.weight", "experts.*.up_proj.weight"]
}
fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)
writer._add_tensor("talker.experts.gate_up_proj", fused_gate_up)
writer.finalize()
shard_path = os.path.join(tmp_path, "model.bin")
saved_tensors = torch.load(shard_path, map_location="cpu")
assert "talker.experts.gate_up_proj" not in saved_tensors
assert "talker.experts.*.gate_proj.weight" not in saved_tensors
assert "talker.experts.0.gate_proj.weight" in saved_tensors
assert "talker.experts.0.up_proj.weight" in saved_tensors
assert torch.equal(saved_tensors["talker.experts.0.gate_proj.weight"], fused_gate_up[0, :3, :])
assert torch.equal(saved_tensors["talker.experts.0.up_proj.weight"], fused_gate_up[0, 3:, :])
def test_do_not_expand_fused_experts_outside_skipped_prefixes(tmp_path, monkeypatch):
model = _FusedExpertsModel()
writer = _make_writer(model, str(tmp_path), monkeypatch)
fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)
writer._add_tensor("block.experts.gate_up_proj", fused_gate_up)
writer.finalize()
shard_path = os.path.join(tmp_path, "model.bin")
saved_tensors = torch.load(shard_path, map_location="cpu")
assert "block.experts.gate_up_proj" in saved_tensors
assert "block.experts.0.gate_proj.weight" not in saved_tensors
def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path, monkeypatch):
model = _DiffusionStyleModel()
model.transformer_blocks[0].linear._parameters["weight"] = model.transformer_blocks[0].linear.weight.to("cpu")
writer = _make_writer(model, str(tmp_path), monkeypatch)
writer.save_module(model.transformer_blocks[0], "transformer_blocks.0")
writer.finalize()
offloaded_weight = model.transformer_blocks[0].linear._parameters["weight"]
assert isinstance(offloaded_weight, torch.nn.Parameter)
assert offloaded_weight.device.type == "meta"
def test_default_max_shard_size_is_fixed(tmp_path, monkeypatch):
writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)
assert writer.max_shard_size == 1 * 1024**2
ShardWriter.reset()
compress_context = SimpleNamespace(formats=[_FormatStub()], output_dir=str(tmp_path))
model_context = SimpleNamespace(is_diffusion=False)
monkeypatch.setattr(CompressContext, "get_context", classmethod(lambda cls: compress_context))
monkeypatch.setattr(ModelContext, "get_context", classmethod(lambda cls: model_context))
default_writer = ShardWriter(_DiffusionStyleModel(), bits=4, safe_serialization=False)
assert default_writer.max_shard_size == 5 * 1024**3
def test_oversized_tensor_does_not_leave_tiny_preceding_shard(tmp_path, monkeypatch):
writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)
writer.max_shard_size = 1024
writer._add_tensor("small", torch.zeros(1, dtype=torch.uint8))
writer._add_tensor("large", torch.zeros(2048, dtype=torch.uint8))
writer._flush_shard()
assert writer.shard_counter == 1
assert set(writer.current_shard_tensors) == set()
@pytest.mark.parametrize("shard_name", ["model-shard-00001.bin", "model.bin", "model-00001-of-00002.bin"])
@pytest.mark.parametrize("mapped", [False, True])
@pytest.mark.parametrize("safe_serialization", [False, True])
def test_finalize_skips_unpacked_weight_of_resumed_packed_module(
tmp_path, monkeypatch, shard_name, mapped, safe_serialization
):
"""A module packed before a crash must not be written again as fp weights.
The resumed run skips tuning for that module, so the model tree still holds
the original ``nn.Linear``. Its ``weight`` name never matches the packed
``qweight`` recovered from the crashed run's shard, so the name-exact dedup
alone would write both into the final checkpoint.
"""
from auto_round import envs
if safe_serialization:
from safetensors.torch import load_file, save_file
save = save_file
load = load_file
shard_name = shard_name.replace(".bin", ".safetensors")
else:
save = torch.save
load = torch.load
multiple_shards = "-of-" in shard_name
suffix = "safetensors" if safe_serialization else "bin"
prefix = "saved_blocks" if mapped else "transformer_blocks"
# A shard flushed by the crashed run, possibly already renamed by finalize.
save(
{
f"{prefix}.0.linear.qweight": torch.zeros(4, 1, dtype=torch.int32),
f"{prefix}.0.linear.scales": torch.ones(4, 1),
f"{prefix}.0.linear.bias": torch.full((4,), 7.0),
},
os.path.join(tmp_path, shard_name),
)
if multiple_shards:
save(
{
"completed.qweight": torch.ones(2, dtype=torch.int16),
"completed.scale": torch.tensor(1.0, dtype=torch.float64),
"completed.empty": torch.empty(0, 3, dtype=torch.bfloat16),
},
str(tmp_path / f"model-00002-of-00002.{suffix}"),
)
monkeypatch.setattr(envs, "AR_RESUME_DIR", str(tmp_path))
model = _DiffusionStyleModel()
writer = _make_writer(model, str(tmp_path), monkeypatch)
writer.use_safetensors = safe_serialization
writer.shard_suffix = suffix
if mapped:
writer.reverse_checkpoint_conversion_mapping = {r"^transformer_blocks": "saved_blocks"}
writer.finalize()
saved = {}
for name in os.listdir(tmp_path):
if name.endswith(f".{suffix}"):
saved.update(load(os.path.join(tmp_path, name)))
assert f"{prefix}.0.linear.qweight" in saved
assert (
f"{prefix}.0.linear.weight" not in saved
), "the packed module must not also be saved as a floating-point weight"
assert torch.equal(saved[f"{prefix}.0.linear.bias"], torch.full((4,), 7.0))
if multiple_shards:
assert torch.equal(saved["completed.qweight"], torch.ones(2, dtype=torch.int16))
# Modules that were never packed are still saved.
assert "proj_out.weight" in saved
# The index covers recovered packed tensors as well as newly written weights.
with open(tmp_path / f"model.{suffix}.index.json", encoding="utf-8") as index_file:
index = json.load(index_file)
assert index["metadata"]["total_parameters"] == sum(tensor.numel() for tensor in saved.values())
assert index["metadata"]["total_size"] == sum(tensor.nbytes for tensor in saved.values())