PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf regression for large models (#680)
Wenhua Cheng21bb06b63db3
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 49d16562f..d49b7bf16 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -876,6 +876,8 @@ class AutoRound(object):pbar = tqdm(all_to_quantized_module_names)block_names_cnt = len(flatten_list(get_block_names(self.model,True)))clear_mem_freq = len(all_to_quantized_module_names)//block_names_cnt+ if clear_mem_freq == 0:+ clear_mem_freq = 1cnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")@@ -895,26 +897,27 @@ class AutoRound(object):model = model.to("cpu")clear_memory()self.quantize_via_rtn_blockwise(all_to_quantized_module_names)- except Exception:- # Final fallback: warn and use CPU-only quantization- logger.warning("Fallback to CPU. "- "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")- model = model.to("cpu")- clear_memory()- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- import accelerate- accelerate.hooks.remove_hook_from_submodules(model)-- orig_device = self.device- self.device = "cpu"- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)- self.device = orig_device- finally:- # Always remove hooks- for hook in hooks:- hook.remove()- else:- raise+ except RuntimeError as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ # Final fallback: warn and use CPU-only quantization+ logger.warning("Fallback to CPU. "+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")+ model = model.to("cpu")+ clear_memory()+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate+ accelerate.hooks.remove_hook_from_submodules(model)++ orig_device = self.device+ self.device = "cpu"+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self.device = orig_device+ else:+ raise+ finally:+ # Always remove hooks+ for hook in hooks:+ hook.remove()# Move back to CPU and free memorymodel.to("cpu")@@ -1119,6 +1122,8 @@ class AutoRound(object):else:block_names_cnt = len(flatten_list(get_block_names(self.model, True)))clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt+ if clear_mem_freq == 0:+ clear_mem_freq = 1pbar = tqdm(all_to_quantized_module_names)cnt = 1for name in pbar:@@ -1223,6 +1228,8 @@ class AutoRound(object):cnt = 1block_names_cnt = len(flatten_list(get_block_names(self.model, True)))clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt+ if clear_mem_freq == 0:+ clear_mem_freq = 1# Process remaining layers not in blocksfor name in all_to_quantized_module_names:self.quantize_layer_via_rtn(name)diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 6b92695e7..14b8d0993 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -19,7 +19,8 @@ import torchfrom pathlib import Pathimport timefrom auto_round.export.export_to_gguf.convert import ModelBase, ModelType, get_model_architecture-from auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_module+from auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_module, \+ clear_memoryTMP_DIR_NAME = "tmp_dir"@@ -159,8 +160,7 @@ def pack_gguf_layer(m.weight = Noneif hasattr(m, "bias"):m.bias = None- import gc- gc.collect()+ clear_memory()model.last_layer_name_to_block_name.pop(name)if len(model.last_layer_name_to_block_name) == 0:for gguf_model in gguf_model_instance_global:diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 676a2bbf8..2e913fd92 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -23,8 +23,7 @@ from auto_round.utils import (get_device_and_parallelism,set_cuda_visible_devices,logger,- )-+)os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"@@ -301,7 +300,6 @@ def tune(args):set_cuda_visible_devices(args.device)device_str, use_auto_mapping = get_device_and_parallelism(args.device)-import torchif not args.disable_deterministic_algorithms:torch.use_deterministic_algorithms(True, warn_only=True)@@ -449,8 +447,8 @@ def tune(args):model_kwargs=model_kwargs,data_type=args.data_type,disable_opt_rtn=args.disable_opt_rtn,- )-+ )+model_name = args.model.rstrip("/")if model_name.split('/')[-1].strip('.') == "" and "gguf" not in args.format:@@ -569,6 +567,3 @@ def lmms_eval(args):apply_chat_template=False,)return results---