Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

17.4 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Unit tests for auto_round.utils.disk_stream_util (AR_DISK_STREAM_MODEL primitives)."""

import json

import pytest
import torch
import torch.nn as nn
from accelerate import init_empty_weights
from safetensors.torch import save_file

from auto_round.utils import disk_stream_util
from auto_round.utils.disk_stream_util import (
    build_meta_model,
    free_module,
    materialize_module,
    materialize_non_block_params,
    total_resident_bytes,
)


class TestNativeFusedMoeCheckpointDetection:
    @staticmethod
    def _write_checkpoint(tmp_path, tensors):
        checkpoint_dir = tmp_path / "checkpoint"
        checkpoint_dir.mkdir()
        save_file(tensors, checkpoint_dir / "model.safetensors")
        return checkpoint_dir

    def test_detects_paired_three_dimensional_fused_experts(self, tmp_path, monkeypatch):
        checkpoint_dir = self._write_checkpoint(
            tmp_path,
            {
                "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),
                "model.layers.0.experts.down_proj": torch.empty(4, 4, 4),
            },
        )

        real_safe_open = disk_stream_util.safe_open

        class HeaderOnlySafeOpen:
            def __init__(self, *args, **kwargs):
                self._context = real_safe_open(*args, **kwargs)

            def __enter__(self):
                self._file = self._context.__enter__()
                return self

            def __exit__(self, *args):
                return self._context.__exit__(*args)

            def keys(self):
                return self._file.keys()

            def get_slice(self, name):
                return self._file.get_slice(name)

            def get_tensor(self, name):
                raise AssertionError(f"native fused detection read tensor payload: {name}")

        monkeypatch.setattr(disk_stream_util, "safe_open", HeaderOnlySafeOpen)

        assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is True

    def test_detects_pair_split_across_shards(self, tmp_path):
        checkpoint_dir = tmp_path / "checkpoint"
        checkpoint_dir.mkdir()
        gate_name = "model.layers.0.experts.gate_up_proj"
        down_name = "model.layers.0.experts.down_proj"
        save_file({gate_name: torch.empty(4, 8, 4)}, checkpoint_dir / "model-00001-of-00002.safetensors")
        save_file({down_name: torch.empty(4, 4, 4)}, checkpoint_dir / "model-00002-of-00002.safetensors")
        index = {
            "weight_map": {
                gate_name: "model-00001-of-00002.safetensors",
                down_name: "model-00002-of-00002.safetensors",
            }
        }
        (checkpoint_dir / "model.safetensors.index.json").write_text(json.dumps(index))

        assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is True

    def test_malformed_safetensors_is_not_detected(self, tmp_path):
        checkpoint_dir = tmp_path / "checkpoint"
        checkpoint_dir.mkdir()
        (checkpoint_dir / "model.safetensors").write_bytes(b"not a safetensors file")

        assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is False

    @pytest.mark.parametrize(
        "tensors",
        [
            {
                "model.layers.0.experts.gate_up_proj": torch.empty(8, 4),
                "model.layers.0.experts.down_proj": torch.empty(4, 4),
            },
            {"model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4)},
            {
                "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),
                "model.layers.1.experts.down_proj": torch.empty(4, 4, 4),
            },
            {
                "model.layers.0.experts.0.gate_proj.weight": torch.empty(8, 4),
                "model.layers.0.experts.0.down_proj.weight": torch.empty(4, 4),
            },
        ],
    )
    def test_rejects_non_native_fused_layouts(self, tmp_path, tensors):
        checkpoint_dir = self._write_checkpoint(tmp_path, tensors)

        assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is False


class TestMaterializeModuleRoundTrip:
    def test_materialize_then_free_round_trip(self, tiny_opt_model_path):
        model, _tokenizer, index = build_meta_model(tiny_opt_model_path)
        block = model.model.decoder.layers[0]

        for _, tensor in list(block.named_parameters()):
            assert str(tensor.device) == "meta"

        materialize_module(block, "model.decoder.layers.0", index, device="cpu")
        for _, tensor in list(block.named_parameters()):
            assert str(tensor.device) != "meta"
        assert total_resident_bytes(block) > 0

        free_module(block)
        for _, tensor in list(block.named_parameters()):
            assert str(tensor.device) == "meta"
        assert total_resident_bytes(block) == 0

    def test_already_materialized_params_are_left_alone(self, tiny_opt_model_path):
        """Shared/tied weights may already be real by the time materialize_module
        runs on a second block referencing them; it must not touch (or crash on)
        a parameter that's already off meta."""
        model, _tokenizer, index = build_meta_model(tiny_opt_model_path)
        block = model.model.decoder.layers[0]
        materialize_module(block, "model.decoder.layers.0", index, device="cpu")
        weight_before = block.self_attn.k_proj.weight.data.clone()

        # Re-materializing an already-real block must be a no-op, not an error.
        materialize_module(block, "model.decoder.layers.0", index, device="cpu")
        assert torch.equal(block.self_attn.k_proj.weight.data, weight_before)

    def test_checkpoint_backed_cpu_buffer_is_overwritten(self):
        """Checkpoint data must replace a CPU buffer's constructor default."""

        class _CpuBufferModule(nn.Module):
            def __init__(self):
                super().__init__()
                self.layer_scalar = nn.Buffer(torch.ones(1))

        with init_empty_weights():
            module = _CpuBufferModule()
        assert module.layer_scalar.device.type == "cpu"
        assert torch.equal(module.layer_scalar, torch.ones(1))

        class _FakeIndex:
            def has_tensor(self, name):
                return name == "block.layer_scalar"

            def read_tensors(self, names, device="cpu"):
                assert names == ["block.layer_scalar"]
                return {"block.layer_scalar": torch.tensor([0.25])}

        materialize_module(module, "block", _FakeIndex(), device="cpu")

        assert torch.equal(module.layer_scalar, torch.tensor([0.25]))


class TestMaterializeModuleDtype:
    """Regression coverage for the dtype-promotion bug: materialize_module used
    to always force the checkpoint's raw on-disk dtype, fighting whatever compute
    dtype the caller had already promoted the (still-meta) model to."""

    def test_prefers_declared_meta_dtype_over_checkpoint_dtype(self, tiny_opt_model_path):
        model, _tokenizer, index = build_meta_model(tiny_opt_model_path)
        block = model.model.decoder.layers[0]

        # Simulate ModelContext._set_amp_dtype()'s `model.to(amp_dtype)` promoting
        # the still-meta block to bf16, even though the checkpoint itself is fp16
        # (facebook/opt-125m's native dtype).
        block.to(torch.bfloat16)
        for _, tensor in block.named_parameters():
            assert tensor.dtype == torch.bfloat16

        materialize_module(block, "model.decoder.layers.0", index, device="cpu")
        for name, tensor in block.named_parameters():
            assert tensor.dtype == torch.bfloat16, f"{name} was materialized as {tensor.dtype}, expected bfloat16"

    def test_falls_back_to_checkpoint_dtype_when_meta_declared_float32(self, tiny_opt_model_path):
        """The one case the checkpoint's dtype must still win: a meta skeleton
        built without an enclosing dtype context (e.g. a module built under
        `torch.device("meta")` alone) defaults to float32 regardless of the
        checkpoint's real dtype."""
        with init_empty_weights():
            linear = nn.Linear(64, 64, bias=False)
        assert linear.weight.dtype == torch.float32

        from auto_round.utils.disk_stream_util import SafetensorsIndex

        with torch.no_grad():
            real_weight = torch.randn(64, 64, dtype=torch.float16)

        class _FakeIndex(SafetensorsIndex):
            def __init__(self):
                self.weight_map = {"fc.weight": "dummy"}

            def has_tensor(self, name):
                return name in self.weight_map

            def read_tensors(self, names, device="cpu"):
                return {n: real_weight.clone() for n in names}

        materialize_module(linear, "fc", _FakeIndex(), device="cpu")
        assert linear.weight.dtype == torch.float16


class TestBuildMetaModel:
    def test_non_block_params_are_real_blocks_stay_meta(self, tiny_opt_model_path):
        model, tokenizer, index = build_meta_model(tiny_opt_model_path)
        assert tokenizer is not None

        block_names = ["model.decoder.layers.0", "model.decoder.layers.1"]
        materialize_non_block_params(model, block_names, index, device="cpu")

        # Embeddings (a non-block module) must be real now.
        assert str(model.model.decoder.embed_tokens.weight.device) != "meta"
        # Decoder blocks must still be untouched (meta).
        for name, tensor in model.model.decoder.layers[0].named_parameters():
            assert str(tensor.device) == "meta", f"{name} was unexpectedly materialized"

    def test_tied_lm_head_shares_the_embedding_tensor(self, tiny_opt_model_path):
        """A tied lm_head.weight has no checkpoint tensor of its own; the meta
        skeleton must tie it to embed_tokens.weight so materializing the
        embedding makes both real (otherwise it stays a separate meta tensor
        and the first device move crashes)."""
        model, _tokenizer, index = build_meta_model(tiny_opt_model_path)
        if not getattr(model.config, "tie_word_embeddings", False):
            pytest.skip("fixture model does not tie word embeddings")
        assert model.lm_head.weight is model.model.decoder.embed_tokens.weight

        block_names = ["model.decoder.layers.0", "model.decoder.layers.1"]
        materialize_non_block_params(model, block_names, index, device="cpu")
        assert str(model.lm_head.weight.device) != "meta", "tied lm_head stayed on meta"


class TestCheckpointNameAliases:
    """Model-side names that differ from checkpoint-side tensor names must
    still materialize, resolving through transformers' checkpoint-conversion
    renames (the router stored as ``mlp.router.gate`` behind a ``mlp.gate``
    module, ``shared_mlp`` tensors behind ``shared_experts``)."""

    def _make_index(self, tensors, tmp_path):
        import json

        from auto_round.utils.disk_stream_util import SafetensorsIndex

        checkpoint_dir = tmp_path / "ckpt"
        checkpoint_dir.mkdir(exist_ok=True)
        (checkpoint_dir / "config.json").write_text(json.dumps({"model_type": "hy_v3"}))

        class _AliasIndex(SafetensorsIndex):
            def __init__(self):
                self.checkpoint_dir = checkpoint_dir
                self.weight_map = {name: "dummy" for name in tensors}
                self._tensors = tensors

            def has_tensor(self, name):
                return name in self._tensors

            def read_tensors(self, names, device="cpu"):
                return {n: self._tensors[n].clone() for n in names}

        return _AliasIndex()

    def _make_block(self):
        import torch.nn as nn

        class _Shared(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate_proj = nn.Linear(4, 4, bias=False)
                self.up_proj = nn.Linear(4, 4, bias=False)

        class _Mlp(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate = nn.Linear(4, 4, bias=False)
                self.shared_experts = _Shared()
                self.up_proj = nn.Linear(4, 4, bias=False)

        with init_empty_weights():
            return _Mlp()

    def test_aliased_names_materialize_from_checkpoint_names(self, tmp_path):
        with torch.no_grad():
            tensors = {
                "model.layers.1.mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16),
                "model.layers.1.mlp.shared_mlp.gate_proj.weight": torch.randn(4, 4, dtype=torch.float16),
                "model.layers.1.mlp.shared_mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),
            }
        block = self._make_block()
        materialize_module(block, "model.layers.1.mlp", self._make_index(tensors, tmp_path), device="cpu")
        assert block.gate.weight.device.type != "meta"
        assert block.shared_experts.gate_proj.weight.device.type != "meta"
        assert block.shared_experts.up_proj.weight.device.type != "meta"

    def test_exact_names_still_resolve_without_alias(self, tmp_path):
        with torch.no_grad():
            tensors = {"model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16)}
        block = self._make_block()
        materialize_module(block, "model.layers.1.mlp", self._make_index(tensors, tmp_path), device="cpu")
        assert block.up_proj.weight.device.type != "meta"

    def test_non_block_params_use_aliases_too(self, tmp_path):
        import torch.nn as nn

        class _Top(nn.Module):
            def __init__(self):
                super().__init__()
                self.mlp = nn.Module()
                self.mlp.gate = nn.Linear(4, 4, bias=False)

        with torch.no_grad():
            tensors = {"mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16)}
        with init_empty_weights():
            top = _Top()
        materialize_non_block_params(top, ["unrelated.block"], self._make_index(tensors, tmp_path), device="cpu")
        assert top.mlp.gate.weight.device.type != "meta"


class TestOffloadLoadBlockAliases:
    """The offloader's block loader must resolve checkpoint-conversion renames
    for meta-skeleton blocks (disk-streamed models), matching what the scoring
    streams do -- otherwise aliased tensors (router, shared experts) silently
    stay on meta and crash the tuning loop's device placement."""

    def _write_checkpoint(self, tmp_path, tensors):
        import json

        from safetensors.torch import save_file

        model_dir = tmp_path / "offload_ckpt"
        model_dir.mkdir(exist_ok=True)
        (model_dir / "config.json").write_text(json.dumps({"model_type": "hy_v3"}))
        save_file(tensors, str(model_dir / "model.safetensors"))
        weight_map = {name: "model.safetensors" for name in tensors}
        (model_dir / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map}))
        return str(model_dir)

    def _make_block(self):
        import torch.nn as nn

        class _Shared(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate_proj = nn.Linear(4, 4, bias=False)
                self.up_proj = nn.Linear(4, 4, bias=False)

        class _Mlp(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate = nn.Linear(4, 4, bias=False)
                self.shared_experts = _Shared()
                self.up_proj = nn.Linear(4, 4, bias=False)

        with init_empty_weights():
            return _Mlp()

    def test_aliased_meta_block_materializes(self, tmp_path):
        from auto_round.utils.offload import load_block_from_model_files

        with torch.no_grad():
            tensors = {
                "model.layers.1.mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16),
                "model.layers.1.mlp.shared_mlp.gate_proj.weight": torch.randn(4, 4, dtype=torch.float16),
                "model.layers.1.mlp.shared_mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),
                "model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),
            }
        model_dir = self._write_checkpoint(tmp_path, tensors)
        block = self._make_block()
        load_block_from_model_files(model_dir, "model.layers.1.mlp", block)
        assert block.gate.weight.device.type != "meta"
        assert block.shared_experts.gate_proj.weight.device.type != "meta"
        assert block.shared_experts.up_proj.weight.device.type != "meta"
        assert block.up_proj.weight.device.type != "meta"
        assert torch.equal(block.gate.weight, tensors["model.layers.1.mlp.router.gate.weight"])

    def test_real_block_still_uses_prefix_loading(self, tmp_path):
        from auto_round.utils.offload import load_block_from_model_files

        with torch.no_grad():
            tensors = {"model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float32)}
        model_dir = self._write_checkpoint(tmp_path, tensors)
        block = self._make_block()
        load_block_from_model_files(model_dir, "model.layers.1.mlp", block)
        assert block.up_proj.weight.device.type != "meta"
        assert torch.equal(block.up_proj.weight, tensors["model.layers.1.mlp.up_proj.weight"])