PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
rm gc collect in packing (#438)
Wenhua Cheng082f01e7f07c
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 7fb7d465a..b52192580 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -12,7 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.-# MIT License## Copyright (c) 2023 潘其威(William)@@ -117,7 +116,6 @@ def pack_layer(name, model, layer_config, backend, pbar):qlayer.to(device)pbar.update(1)-def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",**kwargs):"""Export the model to autogptq format to easily leverage cuda kernel."""diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 48a8c96a3..23843e0d8 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -21,7 +21,6 @@ import torchimport torch.nn as nnimport transformers-from auto_round.export.register import register_formatfrom auto_round.utils import get_layer_names_in_block, get_module, logger, set_moduleimport threadpoolctl as tctlimport inspect@@ -29,7 +28,6 @@ from tqdm import tqdmfrom concurrent.futures import ThreadPoolExecutorfrom auto_round.utils import get_autogptq_packing_qlinear-def check_neq_config(config, data_type, bits, group_size, sym):"""Checks if the provided configuration parameters are not equal to the values in the config dictionary.@@ -133,7 +131,6 @@ def pack_layer(name, model, layer_config, backend, pbar):qlayer.pack(layer, scale, zero, None)qlayer.to(device)else:- from ..export_to_awq.utils import clear_memoryscale, zp = layer_config[name]["scale"].to(torch.float32), layer_config[name]["zp"].to(torch.float32)scale = scale.t().contiguous()zp = zp.t().contiguous()@@ -149,7 +146,7 @@ def pack_layer(name, model, layer_config, backend, pbar):)qlayer.to(device)set_module(model, name, qlayer)- clear_memory()+pbar.update(1)diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex eae25c1bf..d3c345887 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -31,12 +31,11 @@ from auto_round.utils import (logger, get_module,extract_block_names_to_str)import copyimport json-from .utils import WQLinear_GEMM, clear_memory+from .utils import WQLinear_GEMMfrom concurrent.futures import ThreadPoolExecutorimport threadpoolctl as tctlfrom tqdm import tqdm-def pack_layer(name, model, layer_config, backend, pbar):with tctl.threadpool_limits(limits=1):pbar.set_description(f"packing {name}")