Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf extension bug (#1116)

Wenhua Chengbd6314965a3f
diff --git a/auto_round/alg_ext.abi3.so b/auto_round/alg_ext.abi3.so
index 699905b5d..0fd94d2a7 100755
Binary files a/auto_round/alg_ext.abi3.so and b/auto_round/alg_ext.abi3.so differ
diff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.so
index e8ed9622a..9923d27e4 100644
Binary files a/auto_round/auto_scheme/default_alg.abi3.so and b/auto_round/auto_scheme/default_alg.abi3.so differ
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index fab7fd346..64a210d07 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -271,7 +271,6 @@ class BaseCompressor(object):
self.momentum = kwargs.pop("momentum", 0.0)
static_kv_dtype = kwargs.pop("static_kv_dtype", None)
static_attention_dtype = kwargs.pop("static_attention_dtype", None)
- model_dtype = kwargs.pop("model_dtype", None)
device = kwargs.pop("device", None)
if envs.AR_USE_MODELSCOPE:
platform = "model_scope"
@@ -1625,6 +1624,9 @@ class BaseCompressor(object):
return inputs, q_inputs
def configure_layer_config(self, enable_gguf_official_mixed: None | bool = True):
+ fill_default_value = True
+ if self.is_auto_scheme:
+ fill_default_value = False
self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
self.model,
self.layer_config,
@@ -1637,6 +1639,7 @@ class BaseCompressor(object):
self.quant_lm_head,
enable_gguf_official_mixed=enable_gguf_official_mixed,
is_mllm=self.mllm,
+ fill_default_value=fill_default_value,
)
def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
@@ -1722,7 +1725,6 @@ class BaseCompressor(object):
clear_memory(device_list=self.device_list)
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed
- self.model = mv_module_from_gpu(self.model)
logger.info("caching done")
if len(all_blocks) > 1:
pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 1e5683623..3d4132229 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -253,6 +253,7 @@ def set_layer_config(
quant_lm_head: bool = False,
enable_gguf_official_mixed: bool = True,
is_mllm: bool = False,
+ fill_default_value=True,
) -> tuple[dict, bool, dict]:
"""
Normalize, validate, and expand layer-specific quantization configs.
@@ -294,6 +295,7 @@ def set_layer_config(
return config
# ---- main logic ----------------------------------------------
+ extra_scheme_keys = ("scale_dtype",)
scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
layer_config = copy.deepcopy(layer_config) or {}
@@ -325,8 +327,15 @@ def set_layer_config(
else:
default_dict = asdict(default_scheme)
default_dict["scale_dtype"] = default_scale_dtype
+
+ # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,
+ # which should not be filled by default q4km again
+ if fill_default_value:
+ tmp_scheme_keys = scheme_keys
+ else:
+ tmp_scheme_keys = extra_scheme_keys
for cfg in layer_config.values():
- for key in scheme_keys:
+ for key in tmp_scheme_keys:
cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
# 5. collect supported modules
diff --git a/auto_round/export/export_to_autoround/utils.py b/auto_round/export/export_to_autoround/utils.py
index ddef22f1e..5e9e081d2 100644
--- a/auto_round/export/export_to_autoround/utils.py
+++ b/auto_round/export/export_to_autoround/utils.py
@@ -32,10 +32,10 @@ def check_neq_config(config: dict, **expected) -> List[str]:
if missing_expected:
raise ValueError(f"Missing expected values for keys: {missing_expected}")
- # 2. Check missing from layer config
- missing_config = [k for k in scheme_keys if k not in config]
- if missing_config:
- raise ValueError(f"Missing config values for keys: {missing_config}")
+ # # 2. Check missing from layer config
+ # missing_config = [k for k in scheme_keys if k not in config] # None
+ # if missing_config:
+ # raise ValueError(f"Missing config values for keys: {missing_config}")
# 3. Collect mismatches
- return [key for key in scheme_keys if config[key] != expected[key] and config[key] is not None]
+ return [key for key in scheme_keys if config.get(key) not in (expected[key], None)]
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index ea080cc5d..96847c2ea 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -280,9 +280,9 @@ def get_qtype_by_layer_config(layer_config, name, data_qtype):
name = name[: -len(".weight")]
if name not in layer_config or layer_config[name]["bits"] >= 16:
return data_qtype
- bits = layer_config[name]["bits"]
- super_bits = layer_config[name]["super_bits"]
- sym = layer_config[name]["sym"]
+ bits = layer_config[name].get("bits")
+ super_bits = layer_config[name].get("super_bits")
+ sym = layer_config[name].get("sym")
if bits == 2:
return gguf.GGMLQuantizationType.Q2_K
if bits == 3:
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 20cbc3934..906e28ce0 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -917,14 +917,19 @@ def check_to_quantized(config):
"""
if isinstance(config, (dict, QuantizationScheme)):
- bits = int(config.get("bits", 16))
- act_bits = int(config.get("act_bits", 16))
+ bits = config.get("bits", None)
+ act_bits = config.get("act_bits", None)
+
elif hasattr(config, "orig_layer"):
- bits = int(config.orig_layer.bits) if hasattr(config.orig_layer, "bits") else 16
- act_bits = int(config.orig_layer.act_bits) if hasattr(config.orig_layer, "act_bits") else 16
+ bits = getattr(config.orig_layer, "bits", None)
+ act_bits = getattr(config.orig_layer, "act_bits", None)
+
else:
- bits = int(config.bits) if hasattr(config, "bits") else 16
- act_bits = int(config.act_bits) if hasattr(config, "act_bits") else 16
+ bits = getattr(config, "bits", None)
+ act_bits = getattr(config, "act_bits", None)
+
+ bits = int(bits) if bits is not None else 16
+ act_bits = int(act_bits) if act_bits is not None else 16
return bits <= 8 or act_bits <= 8
diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py
index c7bec17d0..b488a745b 100644
--- a/test/test_cuda/test_auto_scheme.py
+++ b/test/test_cuda/test_auto_scheme.py
@@ -23,6 +23,31 @@ class TestAutoScheme(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
+ def test_gguf_k_0(self):
+ model_name = "/models/Qwen3-0.6B"
+ target_bits = 5.5
+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q4_K_M", "GGUF:Q8_0"))
+ ar = AutoRound(model=model_name, scheme=scheme, iters=1, enable_alg_ext=True)
+ ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_gguf_k_1(self):
+ model_name = "/models/Qwen3-0.6B"
+ target_bits = 3.5
+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_1"))
+ ar = AutoRound(model=model_name, scheme=scheme, iters=1, enable_alg_ext=True)
+ ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ #
+ def test_embedding_fallback(self):
+ model_name = "/models/Qwen3-0.6B"
+ target_bits = 5.0
+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q4_K_M", "GGUF:Q8_0"))
+ ar = AutoRound(model=model_name, scheme=scheme, iters=1, enable_alg_ext=True)
+ ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
def test_gguf_export(self):
model_name = "/models/Qwen3-0.6B"
target_bits = 3
diff --git a/test/test_cuda/test_scheme.py b/test/test_cuda/test_scheme.py
index d9151a776..1c603c7ed 100644
--- a/test/test_cuda/test_scheme.py
+++ b/test/test_cuda/test_scheme.py
@@ -20,7 +20,7 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- ## tuning tests
+ # Tuning tests
def test_gguf(self):
ar = AutoRound("/models/Qwen3-0.6B", scheme="W2A16", nsamples=1, iters=1)
ar.quantize_and_save(self.save_folder, format="gguf:q4_k_m")