Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

rm low_cpu when loading the model (#676)

Wenhua Cheng2fe91d77375f
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 594f82900..b1e7bd0b3 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -874,11 +874,13 @@ class AutoRound(object):
# Perform quantization using RTN
from tqdm import tqdm
pbar = tqdm(all_to_quantized_module_names)
+ block_names_cnt = len(flatten_list(get_block_names(self.model,True)))
+ clear_mem_freq = len(all_to_quantized_module_names)//block_names_cnt
cnt = 1
for name in pbar:
pbar.set_description(f"Quantizing {name}")
self.quantize_layer_via_rtn(name)
- if self.low_gpu_mem_usage and cnt % 10 == 0:
+ if cnt % clear_mem_freq == 0:
clear_memory()
cnt = 1
cnt += 1
@@ -1112,12 +1114,14 @@ class AutoRound(object):
if has_gguf_k and not self.disable_opt_rtn:
self.quant_rtn_with_imatrix(all_to_quantized_module_names)
else:
+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
pbar = tqdm(all_to_quantized_module_names)
cnt = 1
for name in pbar:
pbar.set_description(f"Quantizing {name}")
self.quantize_layer_via_rtn(name)
- if self.low_gpu_mem_usage and cnt % 10 == 0:
+ if cnt % clear_mem_freq == 0:
clear_memory()
cnt = 1
cnt += 1
@@ -1214,11 +1218,12 @@ class AutoRound(object):
pbar.close()
cnt = 1
-
+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
# Process remaining layers not in blocks
for name in all_to_quantized_module_names:
self.quantize_layer_via_rtn(name)
- if self.low_gpu_mem_usage and cnt % 10 == 0:
+ if cnt % clear_mem_freq == 0:
clear_memory()
cnt = 1
cnt += 1
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index c9688436e..34dd7aea0 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -114,6 +114,11 @@ def pack_layer(name, model, backend):
else:
qlayer.pack(layer, scale, zero, None)
qlayer.to(device)
+ if hasattr(layer,"weight"):
+ layer.weight = None
+ if hasattr(layer,"bias"):
+ layer.bias = None
+
def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index cf8fe66c9..7a78a2f83 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -228,6 +228,10 @@ def pack_layer(layer_name, model, backend):
)
qlayer.to(device)
set_module(model, layer_name, qlayer)
+ if hasattr(layer,"weight"):
+ layer.weight = None
+ if hasattr(layer,"bias"):
+ layer.bias = None
def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 9f04458dd..4ef02662e 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -67,6 +67,11 @@ def pack_layer(name, model, backend):
zeros=zp,
)
set_module(model, name, q_linear)
+ if hasattr(layer,"weight"):
+ layer.weight = None
+ if hasattr(layer,"bias"):
+ layer.bias = None
+
def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 2a044c825..6038802a6 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1306,14 +1306,14 @@ def llm_load_model(
else:
if _use_hpu_compile_mode():
model = model_cls.from_pretrained(
- pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
+ pretrained_model_name_or_path, torch_dtype=torch_dtype,
attn_implementation="eager",
trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None
)
else:
try:
model = model_cls.from_pretrained(
- pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
+ pretrained_model_name_or_path, torch_dtype=torch_dtype,
trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None
)
except OSError as e:
@@ -1321,7 +1321,6 @@ def llm_load_model(
f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry.")
model = model_cls.from_pretrained(
pretrained_model_name_or_path,
- low_cpu_mem_usage=True,
torch_dtype=torch_dtype,
trust_remote_code=False,
device_map="auto" if use_auto_mapping else None)
diff --git a/test/test_cpu/test_act_quantization.py b/test/test_cpu/test_act_quantization.py
index ef5f1e057..f2d3bcb8c 100644
--- a/test/test_cpu/test_act_quantization.py
+++ b/test/test_cpu/test_act_quantization.py
@@ -87,3 +87,4 @@ class TestAutoRoundAct(unittest.TestCase):
act_data_type="fp8"
)
autoround.quantize()
+