PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix critic bug causing GGUF to run on CPU (#1260)
Wenhua Cheng1b7535a07fb0
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex d473e1bf3..5d7ba6371 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -384,10 +384,6 @@ class BaseCompressor(object):self.lr = 1.0 / self.iterselse:self.lr = lr- if self.bits <= 2 and (self.iters < 1000 or not enable_alg_ext):- logger.warning(- "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` "- )# Automatically adjust the disable_opt_rtn option if the user does not explicitly set it.# To avoid None issue, we keep a copy though it's a little ugly@@ -822,6 +818,11 @@ class BaseCompressor(object):if self.group_size == 0 and "fp8" not in self.data_type:logger.warning("`group_size==0` is not supported for data_type other than fp8 ")+ if self.bits <= 2 and (self.iters < 1000 or not self.enable_alg_ext) and self.super_group_size is None:+ logger.warning(+ "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` "+ )+def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs) -> tuple[torch.nn.Module, dict[str, Any]]:diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 4a4d4e1ff..db95d9547 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -674,11 +674,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, modelf"the setting in layer_config {layer_name} "f"could not match any supported gguf format, please have a check.")- else:- logger.warning_once(- f"the setting in layer_config {layer_name} "- f"could not match any supported gguf format, reset to {new_type}"- )+new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]new_type = _search_gguf_type(new_type)if new_type is None:diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex c1add0ed5..70317cf28 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -152,6 +152,7 @@ def pack_gguf_layer(backend,low_cpu_mem_usage=True,model_type=convert_hf_to_gguf.ModelType.TEXT,+ device=device,)]if model_type == convert_hf_to_gguf.ModelType.MMPROJ: