Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

26.3 KB
Raw Blame
import os
import shutil
import sys
from importlib import import_module
from test.helpers import eval_generated_prompt, get_model_path, get_tiny_model, save_tiny_model
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest
import torch
from packaging import version
from transformers import AutoModelForCausalLM, AutoTokenizer

from auto_round import AutoRound
from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig


def test_pack_gguf_layer_clears_global_state_on_error(monkeypatch, tmp_path):
    export_module = import_module("auto_round.export.export_to_gguf.export")
    failure = RuntimeError("packing failed")
    gguf_model = SimpleNamespace(
        model_arch=object(),
        prepare_tensors=MagicMock(side_effect=failure),
    )
    block = MagicMock()
    block.named_modules.return_value = []
    model = SimpleNamespace(last_layer_name_to_block_name={"layer.weight": "model.layers.0"})

    monkeypatch.setattr(export_module, "get_module", lambda *_: block)
    export_module.gguf_model_instance_global = [gguf_model]

    try:
        with pytest.raises(RuntimeError, match="packing failed"):
            export_module.pack_gguf_layer(
                "layer.weight",
                model,
                "gguf:q4_0",
                str(tmp_path),
                {},
                tokenizer=None,
            )

        assert not hasattr(export_module, "gguf_model_instance_global")
    finally:
        export_module._clear_gguf_model_instances()


def test_save_gguf_clears_global_state_on_writer_error():
    export_module = import_module("auto_round.export.export_to_gguf.export")
    gguf_model = SimpleNamespace(
        model_arch=object(),
        fname_out="broken.gguf",
        write=MagicMock(side_effect=RuntimeError("writer failed")),
    )
    export_module.gguf_model_instance_global = [gguf_model]

    try:
        with pytest.raises(RuntimeError, match="writer failed"):
            export_module.save_quantized_as_gguf("unused", model=object())

        assert not hasattr(export_module, "gguf_model_instance_global")
    finally:
        export_module._clear_gguf_model_instances()


def _run_auto_round_cli(monkeypatch, cmd):
    """Run the auto_round CLI in-process by patching ``sys.argv``.

    ``cmd`` contains only the CLI arguments (everything after ``-m auto_round``).
    This replaces the previous ``os.system('python -m auto_round ...')`` calls so
    the code runs in the test process and is measured by coverage.
    """
    from auto_round.cli.main import run

    monkeypatch.setattr(sys, "argv", ["auto_round", *cmd.split()])
    try:
        run()
    except SystemExit as exc:  # argparse help/errors exit; only 0/None is success
        assert exc.code in (0, None), f"cmd line test fail, exit code {exc.code}"


def test_update_module_applies_replacements_for_gguf(monkeypatch):
    from auto_round import special_model_handler

    model = object()
    replaced_model = object()
    calls = []

    def apply_replacements(candidate, **kwargs):
        calls.append((candidate, kwargs))
        return replaced_model

    class GGUFFormat:
        @staticmethod
        def is_gguf():
            return True

    monkeypatch.setattr(special_model_handler, "apply_replacements", apply_replacements)

    result = special_model_handler.update_module(model, formats=[GGUFFormat()], cleanup_original=False)

    assert result is replaced_model
    assert calls == [(model, {"gguf_export": True})]


def test_make_q3_quants_rmse_scale_matches_levels():
    """The refined Q3_K sub-block scale must be the least-squares fit for the returned levels,
    sum(w*x*L) / sum(w*L*L) with w = x**2, as in llama.cpp's make_q3_quants."""
    from auto_round.export.export_to_gguf.packing import make_q3_quants

    torch.manual_seed(0)
    data = torch.randn(4, 16, 16)
    scales, levels = make_q3_quants(data.clone(), bits=3, do_rmse=True)
    levels = levels.to(torch.int8).float()
    weights = data**2
    expected = (weights * data * levels).sum(-1) / (weights * levels * levels).sum(-1)
    torch.testing.assert_close(scales, expected)


class TestGGUF:

    @classmethod
    def setup_class(self):
        self.model_name = get_model_path("Qwen/Qwen2.5-0.5B-Instruct")
        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)

    @classmethod
    def teardown_class(self):
        shutil.rmtree("runs", ignore_errors=True)

    @pytest.fixture(autouse=True)
    def _save_dir(self, tmp_path):
        self.save_dir = str(tmp_path / "saved")
        yield
        shutil.rmtree(self.save_dir, ignore_errors=True)

    def test_q4_0(self, tiny_qwen_model_path):
        bits, group_size, sym = 4, 32, True
        autoround = AutoRound(
            tiny_qwen_model_path,
            bits=bits,
            group_size=group_size,
            sym=sym,
            iters=1,
            data_type="int",
            nsamples=1,
            seqlen=8,
        )
        quantized_model_path = self.save_dir

        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=quantized_model_path, inplace=False, format="gguf:q4_0"
        )
        gguf_file = os.listdir(quantized_model_path)[0]
        assert gguf_file.endswith(".gguf"), "Saved file is not in gguf format"
        # Accuracy test is covered in test_cuda/export/test_gguf_format.py::TestAutoRound::test_q4_0_accuracy

    def test_q2_k_s_routes_data_driven(self, tiny_qwen_model_path):
        autoround = AutoRound(
            tiny_qwen_model_path,
            scheme="gguf:q2_k_s",
            iters=0,
            nsamples=1,
            seqlen=8,
        )

        assert type(autoround).__name__ == "CompressionOrchestrator"
        assert isinstance(autoround.quantize_config, OptimizedRTNConfig)

    def test_func(self):
        bits, group_size, sym = 4, 128, True
        autoround = AutoRound(
            self.model_name,
            iters=0,
            disable_opt_rtn=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=quantized_model_path, inplace=False, format="gguf:q*_1"
        )
        assert autoround.group_size == 32
        assert not autoround.sym
        gguf_file = os.listdir(quantized_model_path)[0]
        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
        eval_generated_prompt(model, self.tokenizer)

    def test_q4_k_m(self, dataloader, tiny_qwen_model_path):
        model_name = tiny_qwen_model_path
        layer_config = {
            "lm_head": {
                "bits": 4,
                "group_size": 32,
                "sym": False,
                "data_type": "int_asym_dq",
                "super_bits": 6,
                "super_group_size": 8,
            },
            "model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8},
            "model.layers.1.mlp.gate_proj": {"bits": 3},
            "model.layers.0.mlp.gate_proj": {"bits": 8},
        }
        autoround = AutoRound(
            model_name,
            layer_config=layer_config,
            iters=0,
            seqlen=16,
            nsamples=8,
            dataset=dataloader,
            disable_opt_rtn=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=quantized_model_path, format="gguf:q4_k_m,fake"
        )
        assert autoround.layer_config["model.layers.1.self_attn.v_proj"]["super_group_size"] == 16
        assert autoround.layer_config["model.layers.1.self_attn.v_proj"]["data_type"] == "int_sym_dq"
        assert autoround.layer_config["model.layers.0.self_attn.v_proj"]["data_type"] == "int_asym_dq"
        assert autoround.model.model.layers[0].self_attn.v_proj.bits == 4
        assert autoround.model.model.layers[1].self_attn.v_proj.bits == 6
        assert autoround.model.model.embed_tokens.bits == 6
        assert autoround.model.model.embed_tokens.group_size == 16
        assert autoround.model.model.layers[1].mlp.gate_proj.bits == 3
        assert autoround.model.model.layers[0].mlp.gate_proj.bits == 8
        assert autoround.layer_config["model.layers.0.mlp.gate_proj"]["mostly"] == "gguf:q8_0"

    def test_all_format(self, monkeypatch, tiny_qwen_model_path):
        model_name = tiny_qwen_model_path
        # for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]:
        for gguf_format in ["gguf:q4_k_m"]:
            _run_auto_round_cli(
                monkeypatch,
                f"--model {model_name} --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}",
            )
            shutil.rmtree("../../tmp_autoround", ignore_errors=True)

            _run_auto_round_cli(
                monkeypatch,
                f"--model {model_name} --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format fake,{gguf_format}",
            )
            shutil.rmtree("../../tmp_autoround", ignore_errors=True)

        # test q2_k_mixed with iters=0 (RTN) on non-MoE model — should still work
        _run_auto_round_cli(
            monkeypatch,
            f"--model {model_name} --bs 16 --iters 0 --disable_opt_rtn --nsamples 1 --seqlen 16 --scheme GGUF:Q2_K_MIXED",
        )
        shutil.rmtree("../../tmp_autoround", ignore_errors=True)

        # test q2_k_mixed with iters=1 on non-MoE model — should fallback to q4_k_m
        _run_auto_round_cli(
            monkeypatch,
            f"--model {model_name} --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format gguf:q2_k_mixed",
        )
        shutil.rmtree("../../tmp_autoround", ignore_errors=True)

    def test_vlm_gguf(self, tiny_qwen_vl_model_path):
        from auto_round import AutoRound

        autoround = AutoRound(
            tiny_qwen_vl_model_path,
            iters=0,
            nsamples=8,
            disable_opt_rtn=True,
            quant_nontext_module=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
        assert "mmproj-model.gguf" in os.listdir(quantized_model_path)
        for file_name in os.listdir(quantized_model_path):
            file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2
            if file_name == "mmproj-model.gguf":
                assert file_size < 60, f"file size {file_size} MB is too large for non-quantized mmproj-model.gguf"
            else:
                assert file_size < 270, f"file size {file_size} MB is too large for non-quantized mmproj-model.gguf"

    def test_vlm_gguf_wo_quant_nontext_module(self, tiny_qwen_vl_model_path):
        from auto_round import AutoRound

        autoround = AutoRound(
            tiny_qwen_vl_model_path,
            iters=0,
            nsamples=8,
            disable_opt_rtn=True,
            quant_nontext_module=False,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
        assert "mmproj-model.gguf" in os.listdir(quantized_model_path)
        for file_name in os.listdir(quantized_model_path):
            file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2
            if file_name == "mmproj-model.gguf":
                assert abs(file_size - 361) < 5.0
            else:
                assert abs(file_size - 264) < 5.0

    def test_mmproj_uses_native_f32_export_when_nontext_is_not_quantized(self, tmp_path, monkeypatch):
        from auto_round.export.export_to_gguf import export
        from auto_round.export.export_to_gguf.config import ModelType

        ftypes = {
            "q4_0": object(),
            "f32": object(),
        }
        wrapper_calls = []

        class FakeConfig:
            model_type = "qwen2_vl"

        class FakeModel:
            name_or_path = str(tmp_path)
            config = FakeConfig()

        class FakeModelClass:
            def __init__(self, **kwargs):
                self.__dict__.update(kwargs)
                self.model_arch = "MMPROJ"

        class FakeModelBase:
            @staticmethod
            def load_hparams(_dir_model, _is_mistral_format):
                return {"architectures": ["FakeArchitecture"], "quantization_config": {"bits": 4}}

        class FakeConversion:
            ModelBase = FakeModelBase

            @staticmethod
            def model_type(model_type):
                return model_type

            @staticmethod
            def get_model_architecture(_hparams, _model_type):
                return "FakeArchitecture"

            @staticmethod
            def get_model_class(_model_architecture, model_type):
                return FakeModelClass

        def fake_wrapper(model_instance, **kwargs):
            wrapper_calls.append(model_instance)
            model_instance.was_wrapped = True
            model_instance.model = kwargs["model"]
            return model_instance

        monkeypatch.setattr(export, "FTYPE_MAP", ftypes)
        monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)
        monkeypatch.setattr(export, "wrapper_model_instance", fake_wrapper)
        monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)

        model_instance = export.create_model_class(
            output_dir=str(tmp_path),
            model=FakeModel(),
            layer_config={},
            backend="gguf:q4_0",
            model_type=ModelType.MMPROJ,
            quant_nontext_module=False,
        )

        assert wrapper_calls == []
        assert not hasattr(model_instance, "model")
        assert model_instance.ftype is ftypes["f32"]
        assert model_instance.fname_out == tmp_path / "mmproj-model.gguf"

    def test_text_still_uses_autoround_wrapper_when_nontext_is_not_quantized(self, tmp_path, monkeypatch):
        from auto_round.export.export_to_gguf import export
        from auto_round.export.export_to_gguf.config import ModelType

        ftypes = {
            "q4_0": object(),
            "f32": object(),
        }
        wrapper_calls = []

        class FakeConfig:
            model_type = "qwen2_vl"

        class FakeModel:
            name_or_path = str(tmp_path)
            config = FakeConfig()

        class FakeModelClass:
            def __init__(self, **kwargs):
                self.__dict__.update(kwargs)
                self.model_arch = "TEXT"

        class FakeModelBase:
            @staticmethod
            def load_hparams(_dir_model, _is_mistral_format):
                return {"architectures": ["FakeArchitecture"], "quantization_config": {"bits": 4}}

        class FakeConversion:
            ModelBase = FakeModelBase

            @staticmethod
            def model_type(model_type):
                return model_type

            @staticmethod
            def get_model_architecture(_hparams, _model_type):
                return "FakeArchitecture"

            @staticmethod
            def get_model_class(_model_architecture, model_type):
                return FakeModelClass

        def fake_wrapper(model_instance, **kwargs):
            wrapper_calls.append((model_instance, kwargs))
            model_instance.was_wrapped = True
            model_instance.model = kwargs["model"]
            return model_instance

        monkeypatch.setattr(export, "FTYPE_MAP", ftypes)
        monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)
        monkeypatch.setattr(export, "wrapper_model_instance", fake_wrapper)
        monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)

        model_instance = export.create_model_class(
            output_dir=str(tmp_path),
            model=FakeModel(),
            layer_config={},
            backend="gguf:q4_0",
            model_type=ModelType.TEXT,
            quant_nontext_module=False,
        )

        assert len(wrapper_calls) == 1
        assert wrapper_calls[0][0] is model_instance
        assert wrapper_calls[0][1]["quant_nontext_module"] is False
        assert model_instance.was_wrapped
        assert isinstance(model_instance.model, FakeModel)
        assert model_instance.ftype is ftypes["q4_0"]
        assert model_instance.fname_out == tmp_path

    def test_qtype_setting(self, tiny_qwen_vl_model_path):
        # Qwen2.5-0.5B-Instruct no output, token_embed q6_k fallbakc to q8_0 336M
        # Qwen3-0.6B output q6_k, token_embed q4_0  448M
        # Qwen3-8B output q6_k, token_embed q4_0 4.5G
        # Llama-3.2-1B-Instruct o output, token_embed q6_k 736M
        from auto_round.compressors.utils import set_layer_config
        from auto_round.export.export_to_gguf.config import ModelType

        model_name = get_model_path("Qwen/Qwen2.5-0.5B-Instruct")
        ar = AutoRound(model=model_name, scheme="gguf:q4_0", iters=0)
        ar.formats = ["gguf:q4_0"]
        ar.layer_config, _, _ = set_layer_config(
            ar.model,
            ar.layer_config,
            ar.scheme,
            ar.scale_dtype,
            ar.supported_types,
            ar.inner_supported_types,
            ar.quant_block_list,
            ar.ignore_layers,
            ar.quant_lm_head,
            enable_gguf_official_mixed=True,
            is_mllm=ar.mllm,
        )
        assert ar.layer_config["model.embed_tokens"]["bits"] == 8
        assert "lm_head" not in ar.layer_config

        model_name = get_model_path("Qwen/Qwen3-0.6B")
        ar = AutoRound(model=model_name, scheme="gguf:q4_0", iters=0)
        ar.formats = ["gguf:q4_0"]
        ar.layer_config, _, _ = set_layer_config(
            ar.model,
            ar.layer_config,
            ar.scheme,
            ar.scale_dtype,
            ar.supported_types,
            ar.inner_supported_types,
            ar.quant_block_list,
            ar.ignore_layers,
            ar.quant_lm_head,
            enable_gguf_official_mixed=True,
            is_mllm=ar.mllm,
        )
        assert ar.layer_config["model.embed_tokens"]["bits"] == 4
        assert ar.layer_config["lm_head"]["bits"] == 6 and ar.layer_config["lm_head"]["super_bits"] == 8

        layer_config = {
            "model.embed_tokens": {"bits": 6, "super_bits": 8},
            "lm_head": {"bits": 4},
        }
        ar = AutoRound(model=model_name, scheme="gguf:q4_0", iters=0, layer_config=layer_config)
        ar.formats = ["gguf:q4_0"]
        ar.layer_config, _, _ = set_layer_config(
            ar.model,
            ar.layer_config,
            ar.scheme,
            ar.scale_dtype,
            ar.supported_types,
            ar.inner_supported_types,
            ar.quant_block_list,
            ar.ignore_layers,
            ar.quant_lm_head,
            enable_gguf_official_mixed=True,
            is_mllm=ar.mllm,
        )
        assert (
            ar.layer_config["lm_head"]["bits"] == 4
            and ar.layer_config["model.embed_tokens"]["bits"] == 6
            and ar.layer_config["model.embed_tokens"]["super_bits"] == 8
        )

        ar = AutoRound(model=tiny_qwen_vl_model_path, scheme="gguf:q4_0", iters=0)
        ar.formats = ["gguf:q4_0"]
        ar.layer_config, _, _ = set_layer_config(
            ar.model,
            ar.layer_config,
            ar.scheme,
            ar.scale_dtype,
            ar.supported_types,
            ar.inner_supported_types,
            ar.quant_block_list,
            ar.ignore_layers,
            ar.quant_lm_head,
            enable_gguf_official_mixed=True,
            is_mllm=ar.mllm,
        )
        assert ar.layer_config["model.language_model.embed_tokens"]["bits"] == 6
        assert ar.layer_config["model.language_model.embed_tokens"]["super_bits"] == 8

        ar = AutoRound(
            model=tiny_qwen_vl_model_path,
            format="gguf:q4_0",
            iters=0,
            disable_opt_rtn=True,
            disable_model_free=True,
            quant_nontext_module=False,
        )
        ar.post_init()
        assert ar.layer_config["model.language_model.embed_tokens"]["bits"] == 6
        assert ar.layer_config["model.language_model.embed_tokens"]["super_bits"] == 8

    def test_q2k_mixed(self, tiny_qwen_moe_model_path):
        model_name = tiny_qwen_moe_model_path
        autoround = AutoRound(
            model_name,
            iters=0,
            nsamples=1,
            seqlen=16,
            disable_opt_rtn=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")
        gguf_file = os.listdir(quantized_model_path)[0]
        file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2
        assert file_size < 1150, f"file size {file_size} MB is too large for q2_k_mixed format"
        from gguf.gguf_reader import GGUFReader

        gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
        assert gguf_model.get_tensor(2).name == "blk.0.attn_k.weight"
        assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"
        tensor_types = {tensor.name: tensor.tensor_type.name for tensor in gguf_model.tensors}
        assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"

    def test_q2k_mixed_keeps_only_three_dim_expert_weights_at_q2k(self, tiny_qwen_moe_model_path):
        model_name = tiny_qwen_moe_model_path
        autoround = AutoRound(
            model_name,
            iters=0,
            nsamples=1,
            seqlen=16,
            disable_opt_rtn=True,
        )
        _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_mixed")
        gguf_file = os.listdir(quantized_model_path)[0]
        from gguf.gguf_reader import GGUFReader

        gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
        tensor_types = {tensor.name: tensor.tensor_type.name for tensor in gguf_model.tensors}

        assert tensor_types["token_embd.weight"] == "Q8_0"
        assert tensor_types["output.weight"] == "Q8_0"
        assert tensor_types["blk.0.attn_k.weight"] == "Q4_K"
        assert tensor_types["blk.0.ffn_down_exps.weight"] == "Q5_0"
        assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"


class TestGGUFQ5K:
    """Q5_K blocks must round-trip through gguf's reference dequantizer."""

    def _dequantize(self, packed, qtype, shape):
        gguf_quants = pytest.importorskip("gguf.quants")
        from gguf import GGMLQuantizationType

        return torch.from_numpy(gguf_quants.dequantize(packed, GGMLQuantizationType[qtype.upper()])).reshape(shape)

    def test_q5_k_pack_matches_fake_quant(self):
        from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
        from auto_round.export.export_to_gguf.packing import ggml_quant

        torch.manual_seed(0)
        tensor = torch.randn(4, 256)
        qdq, scales, mins = quant_tensor_gguf_asym_dq(tensor, bits=5, scale_dtype=torch.float32)
        packed = ggml_quant(
            qdq.clone(),
            "q5_k",
            scale=scales["scale"],
            wmin=mins["wmin"],
            d_scale=scales["d_scale"],
            d_wmin=mins["d_wmin"],
            device="cpu",
        )
        dequantized = self._dequantize(packed, "q5_k", tensor.shape)
        torch.testing.assert_close(dequantized, qdq, atol=1e-2, rtol=0)

    def test_q5_k_pack_without_stored_params_uses_5_bits(self):
        from auto_round.export.export_to_gguf.packing import ggml_quant

        torch.manual_seed(0)
        tensor = torch.randn(4, 256)
        mse = {}
        for qtype in ("q4_k", "q5_k"):
            packed = ggml_quant(tensor.clone(), qtype, device="cpu")
            mse[qtype] = ((self._dequantize(packed, qtype, tensor.shape) - tensor) ** 2).mean().item()
        # One extra bit should roughly quarter the error.
        assert mse["q5_k"] < mse["q4_k"] / 2, mse


class TestGGUFZeroBlock:
    """All-zero blocks (e.g. padded/unused vocab rows in an embedding tensor) must
    quantize with scale d=0.0, not NaN. A single NaN fp16 `d` in an exported GGUF
    makes llama.cpp return NaN logits for any batch touching those rows."""

    def test_q6_k_all_zero_block_no_nan(self):
        from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq

        tensor = torch.zeros(2, 256)
        tensor[1] = torch.randn(256)
        qdq, scales, _ = quant_tensor_gguf_sym_dq(tensor, bits=6, scale_dtype=torch.float32)
        assert not torch.isnan(scales["scale"]).any(), "Q6_K sub-scales contain NaN for all-zero block"
        assert not torch.isnan(scales["d_scale"]).any(), "Q6_K d_scale contains NaN for all-zero block"
        assert not torch.isnan(qdq).any()
        assert torch.equal(qdq[0], torch.zeros(256)), "all-zero block must reconstruct exactly to zeros"

    def test_q6_k_all_zero_block_packs_zero_d(self):
        import numpy as np

        from auto_round.export.export_to_gguf.packing import ggml_quant

        tensor = torch.zeros(2, 256)
        tensor[1] = torch.randn(256)
        packed = ggml_quant(tensor, "q6_k", device="cpu")
        # Q6_K block layout: [ql(128), qh(64), scales(16), d(fp16)]
        d = np.ascontiguousarray(packed.reshape(2, 210)[:, -2:]).view(np.float16)
        assert not np.isnan(d).any(), f"packed Q6_K d scales contain NaN: {d}"
        assert d[0] == 0.0


class TestGGUFPositiveGroup:
    """K-quant groups whose values are all positive must keep 0 in the range (as llama.cpp
    does), because Q2_K/Q4_K/Q5_K can only store non-negative mins."""

    @pytest.mark.parametrize("qtype, bits", [("q2_k", 2), ("q4_k", 4)])
    def test_all_positive_groups(self, qtype, bits):
        gguf_quants = pytest.importorskip("gguf.quants")
        from gguf import GGMLQuantizationType

        from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
        from auto_round.export.export_to_gguf.packing import ggml_quant

        torch.manual_seed(0)
        tensor = torch.randn(2, 256)
        tensor[1] = torch.rand(256) + 0.5

        qdq, _, _ = quant_tensor_gguf_asym_dq(tensor.clone(), bits=bits, scale_dtype=torch.float32)
        # With 0 kept in the range the MSE is about 0.014 (2-bit) or 0.0007 (4-bit);
        # with a positive min that cannot be stored it is above 0.06.
        assert ((qdq[1] - tensor[1]) ** 2).mean() < 0.03

        packed = ggml_quant(tensor.clone(), qtype, device="cpu")
        dequantized = gguf_quants.dequantize(packed, GGMLQuantizationType[qtype.upper()])
        torch.testing.assert_close(torch.from_numpy(dequantized).reshape(tensor.shape), qdq, atol=1e-2, rtol=0)