PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
rm low_cpu when loading the model (#676)
Wenhua Cheng2fe91d77375f
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 594f82900..b1e7bd0b3 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -874,11 +874,13 @@ class AutoRound(object):# Perform quantization using RTNfrom tqdm import tqdmpbar = tqdm(all_to_quantized_module_names)+ block_names_cnt = len(flatten_list(get_block_names(self.model,True)))+ clear_mem_freq = len(all_to_quantized_module_names)//block_names_cntcnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")self.quantize_layer_via_rtn(name)- if self.low_gpu_mem_usage and cnt % 10 == 0:+ if cnt % clear_mem_freq == 0:clear_memory()cnt = 1cnt += 1@@ -1112,12 +1114,14 @@ class AutoRound(object):if has_gguf_k and not self.disable_opt_rtn:self.quant_rtn_with_imatrix(all_to_quantized_module_names)else:+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cntpbar = tqdm(all_to_quantized_module_names)cnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")self.quantize_layer_via_rtn(name)- if self.low_gpu_mem_usage and cnt % 10 == 0:+ if cnt % clear_mem_freq == 0:clear_memory()cnt = 1cnt += 1@@ -1214,11 +1218,12 @@ class AutoRound(object):pbar.close()cnt = 1-+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt# Process remaining layers not in blocksfor name in all_to_quantized_module_names:self.quantize_layer_via_rtn(name)- if self.low_gpu_mem_usage and cnt % 10 == 0:+ if cnt % clear_mem_freq == 0:clear_memory()cnt = 1cnt += 1diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex c9688436e..34dd7aea0 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -114,6 +114,11 @@ def pack_layer(name, model, backend):else:qlayer.pack(layer, scale, zero, None)qlayer.to(device)+ if hasattr(layer,"weight"):+ layer.weight = None+ if hasattr(layer,"bias"):+ layer.bias = None+def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex cf8fe66c9..7a78a2f83 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -228,6 +228,10 @@ def pack_layer(layer_name, model, backend):)qlayer.to(device)set_module(model, layer_name, qlayer)+ if hasattr(layer,"weight"):+ layer.weight = None+ if hasattr(layer,"bias"):+ layer.bias = Nonedef save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 9f04458dd..4ef02662e 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -67,6 +67,11 @@ def pack_layer(name, model, backend):zeros=zp,)set_module(model, name, q_linear)+ if hasattr(layer,"weight"):+ layer.weight = None+ if hasattr(layer,"bias"):+ layer.bias = None+def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 2a044c825..6038802a6 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1306,14 +1306,14 @@ def llm_load_model(else:if _use_hpu_compile_mode():model = model_cls.from_pretrained(- pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,+ pretrained_model_name_or_path, torch_dtype=torch_dtype,attn_implementation="eager",trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None)else:try:model = model_cls.from_pretrained(- pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,+ pretrained_model_name_or_path, torch_dtype=torch_dtype,trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None)except OSError as e:@@ -1321,7 +1321,6 @@ def llm_load_model(f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry.")model = model_cls.from_pretrained(pretrained_model_name_or_path,- low_cpu_mem_usage=True,torch_dtype=torch_dtype,trust_remote_code=False,device_map="auto" if use_auto_mapping else None)diff --git a/test/test_cpu/test_act_quantization.py b/test/test_cpu/test_act_quantization.pyindex ef5f1e057..f2d3bcb8c 100644--- a/test/test_cpu/test_act_quantization.py+++ b/test/test_cpu/test_act_quantization.py@@ -87,3 +87,4 @@ class TestAutoRoundAct(unittest.TestCase):act_data_type="fp8")autoround.quantize()+