Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix critic bug causing GGUF to run on CPU (#1260)

Wenhua Cheng1b7535a07fb0
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index d473e1bf3..5d7ba6371 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -384,10 +384,6 @@ class BaseCompressor(object):
self.lr = 1.0 / self.iters
else:
self.lr = lr
- if self.bits <= 2 and (self.iters < 1000 or not enable_alg_ext):
- logger.warning(
- "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` "
- )
# Automatically adjust the disable_opt_rtn option if the user does not explicitly set it.
# To avoid None issue, we keep a copy though it's a little ugly
@@ -822,6 +818,11 @@ class BaseCompressor(object):
if self.group_size == 0 and "fp8" not in self.data_type:
logger.warning("`group_size==0` is not supported for data_type other than fp8 ")
+ if self.bits <= 2 and (self.iters < 1000 or not self.enable_alg_ext) and self.super_group_size is None:
+ logger.warning(
+ "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` "
+ )
+
def quantize_and_save(
self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs
) -> tuple[torch.nn.Module, dict[str, Any]]:
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 4a4d4e1ff..db95d9547 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -674,11 +674,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model
f"the setting in layer_config {layer_name} "
f"could not match any supported gguf format, please have a check."
)
- else:
- logger.warning_once(
- f"the setting in layer_config {layer_name} "
- f"could not match any supported gguf format, reset to {new_type}"
- )
+
new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]
new_type = _search_gguf_type(new_type)
if new_type is None:
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index c1add0ed5..70317cf28 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -152,6 +152,7 @@ def pack_gguf_layer(
backend,
low_cpu_mem_usage=True,
model_type=convert_hf_to_gguf.ModelType.TEXT,
+ device=device,
)
]
if model_type == convert_hf_to_gguf.ModelType.MMPROJ: