PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf extension bug (#1116)
Wenhua Chengbd6314965a3f
diff --git a/auto_round/alg_ext.abi3.so b/auto_round/alg_ext.abi3.soindex 699905b5d..0fd94d2a7 100755Binary files a/auto_round/alg_ext.abi3.so and b/auto_round/alg_ext.abi3.so differdiff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.soindex e8ed9622a..9923d27e4 100644Binary files a/auto_round/auto_scheme/default_alg.abi3.so and b/auto_round/auto_scheme/default_alg.abi3.so differdiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex fab7fd346..64a210d07 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -271,7 +271,6 @@ class BaseCompressor(object):self.momentum = kwargs.pop("momentum", 0.0)static_kv_dtype = kwargs.pop("static_kv_dtype", None)static_attention_dtype = kwargs.pop("static_attention_dtype", None)- model_dtype = kwargs.pop("model_dtype", None)device = kwargs.pop("device", None)if envs.AR_USE_MODELSCOPE:platform = "model_scope"@@ -1625,6 +1624,9 @@ class BaseCompressor(object):return inputs, q_inputsdef configure_layer_config(self, enable_gguf_official_mixed: None | bool = True):+ fill_default_value = True+ if self.is_auto_scheme:+ fill_default_value = Falseself.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(self.model,self.layer_config,@@ -1637,6 +1639,7 @@ class BaseCompressor(object):self.quant_lm_head,enable_gguf_official_mixed=enable_gguf_official_mixed,is_mllm=self.mllm,+ fill_default_value=fill_default_value,)def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:@@ -1722,7 +1725,6 @@ class BaseCompressor(object):clear_memory(device_list=self.device_list)if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed- self.model = mv_module_from_gpu(self.model)logger.info("caching done")if len(all_blocks) > 1:pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 1e5683623..3d4132229 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -253,6 +253,7 @@ def set_layer_config(quant_lm_head: bool = False,enable_gguf_official_mixed: bool = True,is_mllm: bool = False,+ fill_default_value=True,) -> tuple[dict, bool, dict]:"""Normalize, validate, and expand layer-specific quantization configs.@@ -294,6 +295,7 @@ def set_layer_config(return config# ---- main logic ----------------------------------------------+ extra_scheme_keys = ("scale_dtype",)scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)layer_config = copy.deepcopy(layer_config) or {}@@ -325,8 +327,15 @@ def set_layer_config(else:default_dict = asdict(default_scheme)default_dict["scale_dtype"] = default_scale_dtype++ # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,+ # which should not be filled by default q4km again+ if fill_default_value:+ tmp_scheme_keys = scheme_keys+ else:+ tmp_scheme_keys = extra_scheme_keysfor cfg in layer_config.values():- for key in scheme_keys:+ for key in tmp_scheme_keys:cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))# 5. collect supported modulesdiff --git a/auto_round/export/export_to_autoround/utils.py b/auto_round/export/export_to_autoround/utils.pyindex ddef22f1e..5e9e081d2 100644--- a/auto_round/export/export_to_autoround/utils.py+++ b/auto_round/export/export_to_autoround/utils.py@@ -32,10 +32,10 @@ def check_neq_config(config: dict, **expected) -> List[str]:if missing_expected:raise ValueError(f"Missing expected values for keys: {missing_expected}")- # 2. Check missing from layer config- missing_config = [k for k in scheme_keys if k not in config]- if missing_config:- raise ValueError(f"Missing config values for keys: {missing_config}")+ # # 2. Check missing from layer config+ # missing_config = [k for k in scheme_keys if k not in config] # None+ # if missing_config:+ # raise ValueError(f"Missing config values for keys: {missing_config}")# 3. Collect mismatches- return [key for key in scheme_keys if config[key] != expected[key] and config[key] is not None]+ return [key for key in scheme_keys if config.get(key) not in (expected[key], None)]diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex ea080cc5d..96847c2ea 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -280,9 +280,9 @@ def get_qtype_by_layer_config(layer_config, name, data_qtype):name = name[: -len(".weight")]if name not in layer_config or layer_config[name]["bits"] >= 16:return data_qtype- bits = layer_config[name]["bits"]- super_bits = layer_config[name]["super_bits"]- sym = layer_config[name]["sym"]+ bits = layer_config[name].get("bits")+ super_bits = layer_config[name].get("super_bits")+ sym = layer_config[name].get("sym")if bits == 2:return gguf.GGMLQuantizationType.Q2_Kif bits == 3:diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 20cbc3934..906e28ce0 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -917,14 +917,19 @@ def check_to_quantized(config):"""if isinstance(config, (dict, QuantizationScheme)):- bits = int(config.get("bits", 16))- act_bits = int(config.get("act_bits", 16))+ bits = config.get("bits", None)+ act_bits = config.get("act_bits", None)+elif hasattr(config, "orig_layer"):- bits = int(config.orig_layer.bits) if hasattr(config.orig_layer, "bits") else 16- act_bits = int(config.orig_layer.act_bits) if hasattr(config.orig_layer, "act_bits") else 16+ bits = getattr(config.orig_layer, "bits", None)+ act_bits = getattr(config.orig_layer, "act_bits", None)+else:- bits = int(config.bits) if hasattr(config, "bits") else 16- act_bits = int(config.act_bits) if hasattr(config, "act_bits") else 16+ bits = getattr(config, "bits", None)+ act_bits = getattr(config, "act_bits", None)++ bits = int(bits) if bits is not None else 16+ act_bits = int(act_bits) if act_bits is not None else 16return bits <= 8 or act_bits <= 8diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.pyindex c7bec17d0..b488a745b 100644--- a/test/test_cuda/test_auto_scheme.py+++ b/test/test_cuda/test_auto_scheme.py@@ -23,6 +23,31 @@ class TestAutoScheme(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)+ def test_gguf_k_0(self):+ model_name = "/models/Qwen3-0.6B"+ target_bits = 5.5+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q4_K_M", "GGUF:Q8_0"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=1, enable_alg_ext=True)+ ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_gguf_k_1(self):+ model_name = "/models/Qwen3-0.6B"+ target_bits = 3.5+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_1"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=1, enable_alg_ext=True)+ ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")+ shutil.rmtree(self.save_dir, ignore_errors=True)++ #+ def test_embedding_fallback(self):+ model_name = "/models/Qwen3-0.6B"+ target_bits = 5.0+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q4_K_M", "GGUF:Q8_0"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=1, enable_alg_ext=True)+ ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")+ shutil.rmtree(self.save_dir, ignore_errors=True)+def test_gguf_export(self):model_name = "/models/Qwen3-0.6B"target_bits = 3diff --git a/test/test_cuda/test_scheme.py b/test/test_cuda/test_scheme.pyindex d9151a776..1c603c7ed 100644--- a/test/test_cuda/test_scheme.py+++ b/test/test_cuda/test_scheme.py@@ -20,7 +20,7 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree(self.save_folder, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- ## tuning tests+ # Tuning testsdef test_gguf(self):ar = AutoRound("/models/Qwen3-0.6B", scheme="W2A16", nsamples=1, iters=1)ar.quantize_and_save(self.save_folder, format="gguf:q4_k_m")