Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

11.1 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
from types import SimpleNamespace

import pytest
import torch

from auto_round.compressors.shard_writer import ShardWriter
from auto_round.context.compress import CompressContext
from auto_round.context.model import ModelContext


class _ToyBlock(torch.nn.Module):

    def __init__(self):
        super().__init__()
        self.linear = torch.nn.Linear(4, 4)


class _DiffusionStyleModel(torch.nn.Module):

    def __init__(self):
        super().__init__()
        self.transformer_blocks = torch.nn.ModuleList([_ToyBlock()])
        self.proj_out = torch.nn.Linear(4, 2)
        self.config = SimpleNamespace(model_type="toy-diffusion")


class _FormatStub:

    def get_backend_name(self):
        return "auto_round"


def _make_writer(model, output_dir, monkeypatch):
    ShardWriter.reset()
    compress_context = SimpleNamespace(formats=[_FormatStub()], output_dir=output_dir)
    model_context = SimpleNamespace(is_diffusion=False)
    monkeypatch.setattr(CompressContext, "get_context", classmethod(lambda cls: compress_context))
    monkeypatch.setattr(ModelContext, "get_context", classmethod(lambda cls: model_context))
    return ShardWriter(model, bits=4, max_shard_size="1MB", safe_serialization=False)


def test_finalize_saves_tail_layer_when_tie_word_embeddings_missing(tmp_path, monkeypatch):
    model = _DiffusionStyleModel()
    writer = _make_writer(model, str(tmp_path), monkeypatch)

    assert writer.lm_head_name == "proj_out"
    assert not hasattr(model.config, "tie_word_embeddings")

    writer.save_module(model.transformer_blocks[0], "transformer_blocks.0")
    writer.finalize()

    shard_path = os.path.join(tmp_path, "model.bin")
    saved_tensors = torch.load(shard_path, map_location="cpu")

    assert "transformer_blocks.0.linear.weight" in saved_tensors
    assert "proj_out.weight" in saved_tensors, "proj_out must be saved when tie_word_embeddings is absent"
    assert "proj_out.bias" in saved_tensors


class _LMStyleModel(torch.nn.Module):
    """Model whose config explicitly sets tie_word_embeddings=True."""

    def __init__(self):
        super().__init__()
        self.transformer_blocks = torch.nn.ModuleList([_ToyBlock()])
        self.lm_head = torch.nn.Linear(4, 2, bias=False)
        self.config = SimpleNamespace(model_type="toy-lm", tie_word_embeddings=True)


class _ToyExperts(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.is_transposed = False


class _FusedExpertsModel(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.block = torch.nn.Module()
        self.block.experts = _ToyExperts()
        self.talker = torch.nn.Module()
        self.talker.experts = _ToyExperts()
        self.config = SimpleNamespace(model_type="qwen3_omni_moe")


def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path, monkeypatch):
    """Complementary test: when tie_word_embeddings=True the lm_head should be
    skipped (not written to disk) and offloaded to meta."""
    model = _LMStyleModel()
    writer = _make_writer(model, str(tmp_path), monkeypatch)

    assert writer.lm_head_name == "lm_head"

    writer.save_module(model.transformer_blocks[0], "transformer_blocks.0")
    writer.finalize()

    shard_path = os.path.join(tmp_path, "model.bin")
    saved_tensors = torch.load(shard_path, map_location="cpu")

    assert "transformer_blocks.0.linear.weight" in saved_tensors
    assert "lm_head.weight" not in saved_tensors, "lm_head must be skipped when tied"
    assert model.lm_head.weight.device.type == "meta"


def test_expand_fused_experts_for_skipped_talker_prefix(tmp_path, monkeypatch):
    """Talker fused 3D weights must be expanded to exact per-expert 2D keys.

    Real Qwen3-Omni-MoE exports attach reverse checkpoint conversion mappings for
    MoE projections. If we apply that mapping to the fused talker tensor before
    expanding it, the fused tensor is saved under a wildcard key such as
    ``talker.experts.*.gate_proj.weight``. That breaks reload because
    transformers expects concrete per-expert 2D keys after save_pretrained.
    """
    model = _FusedExpertsModel()
    writer = _make_writer(model, str(tmp_path), monkeypatch)
    writer.reverse_checkpoint_conversion_mapping = {
        r"experts\.gate_up_proj$": ["experts.*.gate_proj.weight", "experts.*.up_proj.weight"]
    }

    fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)
    writer._add_tensor("talker.experts.gate_up_proj", fused_gate_up)
    writer.finalize()

    shard_path = os.path.join(tmp_path, "model.bin")
    saved_tensors = torch.load(shard_path, map_location="cpu")

    assert "talker.experts.gate_up_proj" not in saved_tensors
    assert "talker.experts.*.gate_proj.weight" not in saved_tensors
    assert "talker.experts.0.gate_proj.weight" in saved_tensors
    assert "talker.experts.0.up_proj.weight" in saved_tensors
    assert torch.equal(saved_tensors["talker.experts.0.gate_proj.weight"], fused_gate_up[0, :3, :])
    assert torch.equal(saved_tensors["talker.experts.0.up_proj.weight"], fused_gate_up[0, 3:, :])


def test_do_not_expand_fused_experts_outside_skipped_prefixes(tmp_path, monkeypatch):
    model = _FusedExpertsModel()
    writer = _make_writer(model, str(tmp_path), monkeypatch)

    fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)
    writer._add_tensor("block.experts.gate_up_proj", fused_gate_up)
    writer.finalize()

    shard_path = os.path.join(tmp_path, "model.bin")
    saved_tensors = torch.load(shard_path, map_location="cpu")

    assert "block.experts.gate_up_proj" in saved_tensors
    assert "block.experts.0.gate_proj.weight" not in saved_tensors


def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path, monkeypatch):
    model = _DiffusionStyleModel()
    model.transformer_blocks[0].linear._parameters["weight"] = model.transformer_blocks[0].linear.weight.to("cpu")
    writer = _make_writer(model, str(tmp_path), monkeypatch)

    writer.save_module(model.transformer_blocks[0], "transformer_blocks.0")
    writer.finalize()

    offloaded_weight = model.transformer_blocks[0].linear._parameters["weight"]
    assert isinstance(offloaded_weight, torch.nn.Parameter)
    assert offloaded_weight.device.type == "meta"


def test_default_max_shard_size_is_fixed(tmp_path, monkeypatch):
    writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)
    assert writer.max_shard_size == 1 * 1024**2

    ShardWriter.reset()
    compress_context = SimpleNamespace(formats=[_FormatStub()], output_dir=str(tmp_path))
    model_context = SimpleNamespace(is_diffusion=False)
    monkeypatch.setattr(CompressContext, "get_context", classmethod(lambda cls: compress_context))
    monkeypatch.setattr(ModelContext, "get_context", classmethod(lambda cls: model_context))
    default_writer = ShardWriter(_DiffusionStyleModel(), bits=4, safe_serialization=False)
    assert default_writer.max_shard_size == 5 * 1024**3


def test_oversized_tensor_does_not_leave_tiny_preceding_shard(tmp_path, monkeypatch):
    writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)
    writer.max_shard_size = 1024

    writer._add_tensor("small", torch.zeros(1, dtype=torch.uint8))
    writer._add_tensor("large", torch.zeros(2048, dtype=torch.uint8))
    writer._flush_shard()

    assert writer.shard_counter == 1
    assert set(writer.current_shard_tensors) == set()


@pytest.mark.parametrize("shard_name", ["model-shard-00001.bin", "model.bin", "model-00001-of-00002.bin"])
@pytest.mark.parametrize("mapped", [False, True])
@pytest.mark.parametrize("safe_serialization", [False, True])
def test_finalize_skips_unpacked_weight_of_resumed_packed_module(
    tmp_path, monkeypatch, shard_name, mapped, safe_serialization
):
    """A module packed before a crash must not be written again as fp weights.

    The resumed run skips tuning for that module, so the model tree still holds
    the original ``nn.Linear``. Its ``weight`` name never matches the packed
    ``qweight`` recovered from the crashed run's shard, so the name-exact dedup
    alone would write both into the final checkpoint.
    """
    from auto_round import envs

    if safe_serialization:
        from safetensors.torch import load_file, save_file

        save = save_file
        load = load_file
        shard_name = shard_name.replace(".bin", ".safetensors")
    else:
        save = torch.save
        load = torch.load

    multiple_shards = "-of-" in shard_name
    suffix = "safetensors" if safe_serialization else "bin"
    prefix = "saved_blocks" if mapped else "transformer_blocks"
    # A shard flushed by the crashed run, possibly already renamed by finalize.
    save(
        {
            f"{prefix}.0.linear.qweight": torch.zeros(4, 1, dtype=torch.int32),
            f"{prefix}.0.linear.scales": torch.ones(4, 1),
            f"{prefix}.0.linear.bias": torch.full((4,), 7.0),
        },
        os.path.join(tmp_path, shard_name),
    )
    if multiple_shards:
        save(
            {
                "completed.qweight": torch.ones(2, dtype=torch.int16),
                "completed.scale": torch.tensor(1.0, dtype=torch.float64),
                "completed.empty": torch.empty(0, 3, dtype=torch.bfloat16),
            },
            str(tmp_path / f"model-00002-of-00002.{suffix}"),
        )

    monkeypatch.setattr(envs, "AR_RESUME_DIR", str(tmp_path))

    model = _DiffusionStyleModel()
    writer = _make_writer(model, str(tmp_path), monkeypatch)
    writer.use_safetensors = safe_serialization
    writer.shard_suffix = suffix
    if mapped:
        writer.reverse_checkpoint_conversion_mapping = {r"^transformer_blocks": "saved_blocks"}
    writer.finalize()

    saved = {}
    for name in os.listdir(tmp_path):
        if name.endswith(f".{suffix}"):
            saved.update(load(os.path.join(tmp_path, name)))

    assert f"{prefix}.0.linear.qweight" in saved
    assert (
        f"{prefix}.0.linear.weight" not in saved
    ), "the packed module must not also be saved as a floating-point weight"
    assert torch.equal(saved[f"{prefix}.0.linear.bias"], torch.full((4,), 7.0))
    if multiple_shards:
        assert torch.equal(saved["completed.qweight"], torch.ones(2, dtype=torch.int16))
    # Modules that were never packed are still saved.
    assert "proj_out.weight" in saved

    # The index covers recovered packed tensors as well as newly written weights.
    with open(tmp_path / f"model.{suffix}.index.json", encoding="utf-8") as index_file:
        index = json.load(index_file)
    assert index["metadata"]["total_parameters"] == sum(tensor.numel() for tensor in saved.values())
    assert index["metadata"]["total_size"] == sum(tensor.nbytes for tensor in saved.values())