Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

gguf fix bug of tensors not on same tensor and gguf:q2_k_mixed (#1263)

Heng Guo2c67712fddfa
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index ed99b1485..ee3d4ecee 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -404,6 +404,7 @@ def prepare_tensors(cls):
modify_name = _special_name_handle(cls, name)
orig_device = data_torch.device
+ data_torch = data_torch.to("cpu")
for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):
skip = False
for tensor_info in cls.gguf_writer.tensors:
diff --git a/auto_round/formats.py b/auto_round/formats.py
index 15d8b34a6..b853dae22 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -76,8 +76,8 @@ def _check_compatibility(formats: list[str], ar: BaseCompressor):
)
gguf_format_name = get_gguf_scheme(ar.scheme)
if gguf_format_name:
- # if gguf_format_name.lower().endswith("mixed"):
- # gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")
+ if gguf_format_name.lower().endswith("mixed"):
+ gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")
if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):
tmp_format_name = gguf_format_name.lower() if "fake" not in formats else f"{gguf_format_name.lower()},fake"
logger.warning(
@@ -669,7 +669,7 @@ class GGUFFormat(OutputFormat):
if format.lower().endswith("_mixed"):
from auto_round.schemes import _handle_special_schemes
- ar.layer_config = _handle_special_schemes(scheme, ar.layer_config, ar.model)
+ ar.layer_config = _handle_special_schemes(gguf_format, ar.layer_config, ar.model)
gguf_format = gguf_format.lower().replace("_mixed", "_s")
if isinstance(scheme, str) and scheme.lower() != gguf_format:
logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 95078742c..93110a833 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -301,6 +301,8 @@ def _handle_special_schemes(scheme_name: str, layer_config: dict, model: torch.n
Provide some special auto_round recipes.
"""
+ if layer_config is None:
+ layer_config = {}
if scheme_name.lower() == "gguf:q2_k_mixed":
for n, m in model.named_modules():
if n in layer_config:
diff --git a/test/fixtures.py b/test/fixtures.py
index c76040322..01ba5cd70 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -29,7 +29,7 @@ def tiny_opt_model_path():
tiny_model_path = "./tmp/tiny_opt_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -38,7 +38,7 @@ def tiny_lamini_model_path():
tiny_model_path = "./tmp/tiny_lamini_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -47,7 +47,7 @@ def tiny_gptj_model_path():
tiny_model_path = "./tmp/tiny_gptj_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -56,7 +56,7 @@ def tiny_phi2_model_path():
tiny_model_path = "./tmp/tiny_phi2_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -65,7 +65,7 @@ def tiny_deepseek_v2_model_path():
tiny_model_path = "./tmp/tiny_deepseek_v2_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -74,7 +74,7 @@ def tiny_gemma_model_path():
tiny_model_path = "./tmp/tiny_gemma_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -83,7 +83,7 @@ def tiny_qwen_model_path():
tiny_model_path = "./tmp/tiny_qwen_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -92,7 +92,7 @@ def tiny_untied_qwen_model_path():
tiny_model_path = "./tmp/tiny_untied_qwen_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, force_untie=True)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -101,7 +101,7 @@ def tiny_qwen_moe_model_path():
tiny_model_path = "./tmp/tiny_qwen_moe_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -110,7 +110,7 @@ def tiny_qwen_vl_model_path():
tiny_model_path = "./tmp/tiny_qwen_vl_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=True)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -119,7 +119,7 @@ def tiny_qwen_2_5_vl_model_path():
tiny_model_path = "./tmp/tiny_qwen_2_5_vl_model_path"
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=True)
yield tiny_model_path
- shutil.rmtree(tiny_model_path)
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(autouse=True, scope="session")
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index feeddcd0f..4ba59629e 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -8,7 +8,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
-from ...helpers import get_model_path, get_tiny_model
+from ...helpers import get_model_path, get_tiny_model, save_tiny_model
AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
@@ -189,14 +189,6 @@ class TestGGUF:
assert False, "cmd line test fail, please have a check"
shutil.rmtree("../../tmp_autoround", ignore_errors=True)
- res = os.system(
- f"PYTHONPATH='AUTO_ROUND_PATH:$PYTHONPATH' {python_path} -m auto_round --model {model_name}"
- f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format gguf:q2_k_mixed"
- )
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
- shutil.rmtree("../../tmp_autoround", ignore_errors=True)
-
def test_vlm_gguf(self):
from ...helpers import save_tiny_model
@@ -292,3 +284,34 @@ class TestGGUF:
and ar.layer_config["model.embed_tokens"]["bits"] == 6
and ar.layer_config["model.embed_tokens"]["super_bits"] == 8
)
+
+ def test_q2k_mixed(self):
+ model_name = get_model_path("Qwen/Qwen1.5-MoE-A2.7B")
+ saved_tiny_model_path = save_tiny_model(
+ model_name,
+ "./tmp/tiny_qwen_model_path",
+ num_layers=3,
+ is_mllm=False,
+ )
+ autoround = AutoRound(
+ saved_tiny_model_path,
+ iters=0,
+ nsamples=1,
+ seqlen=16,
+ disable_opt_rtn=True,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")
+ gguf_file = os.listdir(quantized_model_path)[0]
+ file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2
+ assert abs(file_size - 1362) < 5.0
+ from gguf.gguf_reader import GGUFReader
+
+ gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
+ assert gguf_model.get_tensor(2).name == "blk.0.attn_k.weight"
+ assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"
+ assert gguf_model.get_tensor(10).name == "blk.0.ffn_up_exps.weight"
+ assert gguf_model.get_tensor(10).tensor_type.name == "Q2_K"
+
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree(saved_tiny_model_path, ignore_errors=True)
diff --git a/test/test_cuda/schemes/test_scheme.py b/test/test_cuda/schemes/test_scheme.py
index 739a742a6..315c5984b 100644
--- a/test/test_cuda/schemes/test_scheme.py
+++ b/test/test_cuda/schemes/test_scheme.py
@@ -1,3 +1,4 @@
+import os
import shutil
import pytest
@@ -5,7 +6,7 @@ import pytest
from auto_round import AutoRound
from auto_round.schemes import QuantizationScheme
-from ...helpers import get_model_path
+from ...helpers import get_model_path, save_tiny_model
class TestAutoRound:
@@ -102,3 +103,34 @@ class TestAutoRound:
assert m.bits == 8
if n == "model.decoder.layers.4.self_attn.k_proj":
assert m.group_size == 64
+
+ def test_q2k_mixed(self):
+ model_path = "/data0/MiroThinker-v1.5-30B"
+ saved_tiny_model_path = save_tiny_model(
+ model_path,
+ "./tmp/tiny_qwen_model_path",
+ num_layers=3,
+ is_mllm=False,
+ )
+ autoround = AutoRound(
+ saved_tiny_model_path,
+ iters=0,
+ nsamples=1,
+ seqlen=16,
+ disable_opt_rtn=True,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")
+ gguf_file = os.listdir(quantized_model_path)[0]
+ file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2
+ assert abs(file_size - 1236) < 5.0
+ from gguf.gguf_reader import GGUFReader
+
+ gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
+ assert gguf_model.get_tensor(2).name == "blk.0.attn_v.weight"
+ assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"
+ assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"
+ assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"
+
+ shutil.rmtree(saved_tiny_model_path, ignore_errors=True)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)