PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
gguf fix bug of tensors not on same tensor and gguf:q2_k_mixed (#1263)
Heng Guo2c67712fddfa
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex ed99b1485..ee3d4ecee 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -404,6 +404,7 @@ def prepare_tensors(cls):modify_name = _special_name_handle(cls, name)orig_device = data_torch.device+ data_torch = data_torch.to("cpu")for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):skip = Falsefor tensor_info in cls.gguf_writer.tensors:diff --git a/auto_round/formats.py b/auto_round/formats.pyindex 15d8b34a6..b853dae22 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -76,8 +76,8 @@ def _check_compatibility(formats: list[str], ar: BaseCompressor):)gguf_format_name = get_gguf_scheme(ar.scheme)if gguf_format_name:- # if gguf_format_name.lower().endswith("mixed"):- # gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")+ if gguf_format_name.lower().endswith("mixed"):+ gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):tmp_format_name = gguf_format_name.lower() if "fake" not in formats else f"{gguf_format_name.lower()},fake"logger.warning(@@ -669,7 +669,7 @@ class GGUFFormat(OutputFormat):if format.lower().endswith("_mixed"):from auto_round.schemes import _handle_special_schemes- ar.layer_config = _handle_special_schemes(scheme, ar.layer_config, ar.model)+ ar.layer_config = _handle_special_schemes(gguf_format, ar.layer_config, ar.model)gguf_format = gguf_format.lower().replace("_mixed", "_s")if isinstance(scheme, str) and scheme.lower() != gguf_format:logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 95078742c..93110a833 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -301,6 +301,8 @@ def _handle_special_schemes(scheme_name: str, layer_config: dict, model: torch.nProvide some special auto_round recipes."""+ if layer_config is None:+ layer_config = {}if scheme_name.lower() == "gguf:q2_k_mixed":for n, m in model.named_modules():if n in layer_config:diff --git a/test/fixtures.py b/test/fixtures.pyindex c76040322..01ba5cd70 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -29,7 +29,7 @@ def tiny_opt_model_path():tiny_model_path = "./tmp/tiny_opt_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -38,7 +38,7 @@ def tiny_lamini_model_path():tiny_model_path = "./tmp/tiny_lamini_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -47,7 +47,7 @@ def tiny_gptj_model_path():tiny_model_path = "./tmp/tiny_gptj_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -56,7 +56,7 @@ def tiny_phi2_model_path():tiny_model_path = "./tmp/tiny_phi2_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -65,7 +65,7 @@ def tiny_deepseek_v2_model_path():tiny_model_path = "./tmp/tiny_deepseek_v2_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -74,7 +74,7 @@ def tiny_gemma_model_path():tiny_model_path = "./tmp/tiny_gemma_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -83,7 +83,7 @@ def tiny_qwen_model_path():tiny_model_path = "./tmp/tiny_qwen_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -92,7 +92,7 @@ def tiny_untied_qwen_model_path():tiny_model_path = "./tmp/tiny_untied_qwen_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, force_untie=True)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -101,7 +101,7 @@ def tiny_qwen_moe_model_path():tiny_model_path = "./tmp/tiny_qwen_moe_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -110,7 +110,7 @@ def tiny_qwen_vl_model_path():tiny_model_path = "./tmp/tiny_qwen_vl_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=True)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")@@ -119,7 +119,7 @@ def tiny_qwen_2_5_vl_model_path():tiny_model_path = "./tmp/tiny_qwen_2_5_vl_model_path"tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=True)yield tiny_model_path- shutil.rmtree(tiny_model_path)+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(autouse=True, scope="session")diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex feeddcd0f..4ba59629e 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -8,7 +8,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound-from ...helpers import get_model_path, get_tiny_model+from ...helpers import get_model_path, get_tiny_model, save_tiny_modelAUTO_ROUND_PATH = __file__.split("/")AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])@@ -189,14 +189,6 @@ class TestGGUF:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)- res = os.system(- f"PYTHONPATH='AUTO_ROUND_PATH:$PYTHONPATH' {python_path} -m auto_round --model {model_name}"- f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format gguf:q2_k_mixed"- )- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"- shutil.rmtree("../../tmp_autoround", ignore_errors=True)-def test_vlm_gguf(self):from ...helpers import save_tiny_model@@ -292,3 +284,34 @@ class TestGGUF:and ar.layer_config["model.embed_tokens"]["bits"] == 6and ar.layer_config["model.embed_tokens"]["super_bits"] == 8)++ def test_q2k_mixed(self):+ model_name = get_model_path("Qwen/Qwen1.5-MoE-A2.7B")+ saved_tiny_model_path = save_tiny_model(+ model_name,+ "./tmp/tiny_qwen_model_path",+ num_layers=3,+ is_mllm=False,+ )+ autoround = AutoRound(+ saved_tiny_model_path,+ iters=0,+ nsamples=1,+ seqlen=16,+ disable_opt_rtn=True,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")+ gguf_file = os.listdir(quantized_model_path)[0]+ file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2+ assert abs(file_size - 1362) < 5.0+ from gguf.gguf_reader import GGUFReader++ gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))+ assert gguf_model.get_tensor(2).name == "blk.0.attn_k.weight"+ assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"+ assert gguf_model.get_tensor(10).name == "blk.0.ffn_up_exps.weight"+ assert gguf_model.get_tensor(10).tensor_type.name == "Q2_K"++ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(saved_tiny_model_path, ignore_errors=True)diff --git a/test/test_cuda/schemes/test_scheme.py b/test/test_cuda/schemes/test_scheme.pyindex 739a742a6..315c5984b 100644--- a/test/test_cuda/schemes/test_scheme.py+++ b/test/test_cuda/schemes/test_scheme.py@@ -1,3 +1,4 @@+import osimport shutilimport pytest@@ -5,7 +6,7 @@ import pytestfrom auto_round import AutoRoundfrom auto_round.schemes import QuantizationScheme-from ...helpers import get_model_path+from ...helpers import get_model_path, save_tiny_modelclass TestAutoRound:@@ -102,3 +103,34 @@ class TestAutoRound:assert m.bits == 8if n == "model.decoder.layers.4.self_attn.k_proj":assert m.group_size == 64++ def test_q2k_mixed(self):+ model_path = "/data0/MiroThinker-v1.5-30B"+ saved_tiny_model_path = save_tiny_model(+ model_path,+ "./tmp/tiny_qwen_model_path",+ num_layers=3,+ is_mllm=False,+ )+ autoround = AutoRound(+ saved_tiny_model_path,+ iters=0,+ nsamples=1,+ seqlen=16,+ disable_opt_rtn=True,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")+ gguf_file = os.listdir(quantized_model_path)[0]+ file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2+ assert abs(file_size - 1236) < 5.0+ from gguf.gguf_reader import GGUFReader++ gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))+ assert gguf_model.get_tensor(2).name == "blk.0.attn_v.weight"+ assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"+ assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"+ assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"++ shutil.rmtree(saved_tiny_model_path, ignore_errors=True)+ shutil.rmtree(quantized_model_path, ignore_errors=True)