PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix cuda ut (#592)
Wenhua Cheng4c3dc07346a6
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b8ee17dd0..03acf72cc 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -167,6 +167,8 @@ class AutoRound(object):model_kwargs: dict = None,**kwargs,):+ if kwargs:+ logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")self.quantized = Falseself.model_orig_dtype = model.dtypeself.seed = seeddiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 42fc63e6d..b40c7fe2f 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -20,6 +20,7 @@ import osimport torchimport torch.nn as nnimport transformers+try:import auto_round_extension.triton.qlinear_tritonv2except Exception as error:@@ -27,12 +28,12 @@ except Exception as error:import auto_round_extension.torch.qlinear_torchimport auto_round.export.export_to_autoround.qlinear_triton_actfrom auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \- filter_quantization_config+ filter_quantization_config, SUPPORTED_FORMATSimport threadpoolctl as tctlimport inspectfrom tqdm import tqdmfrom concurrent.futures import ThreadPoolExecutor-from auto_round.utils import get_autogptq_packing_qlinear,check_start_with_block_name+from auto_round.utils import get_autogptq_packing_qlinear, check_start_with_block_namedef check_neq_config(config, data_type, bits, group_size, sym):@@ -78,22 +79,20 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_return auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinearfrom auto_round_extension.torch.qlinear_torch import QuantLinearreturn QuantLinear- elif "auto_round" in backend and "gptq" in backend:+ elif "gptqmodel" in backend:+ from auto_round_extension.torch.qlinear_torch import QuantLinear+ return QuantLinear+ elif "auto_round" in backend and "gptq" in backend and "gptqmodel" not in backend:from auto_round_extension.torch.qlinear_torch_zp import QuantLinearreturn QuantLinearelif "awq" in backend:from ..export_to_awq.utils import WQLinear_GEMMreturn WQLinear_GEMM- elif "gptqmodel" in backend:- from auto_round_extension.torch.qlinear_torch import QuantLinear- return QuantLinearelif "gptq" in backend and not "gptqmodel" in backend: ## have g_idxreturn get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:raise ValueError(- f"Unsupported backend: '{backend}'. "- "Only 'auto_round', 'auto_round:auto_awq', 'auto_round:auto_gptq', 'awq', and 'gptq' are supported."- )+ f"unsupported backend: '{backend}'. Supported backends are: {', '.join(SUPPORTED_FORMATS)}")def pack_qact_layer(name, model):@@ -197,9 +196,7 @@ def pack_layer(layer_name, model, backend):new_layer.device = deviceset_module(model, layer_name, new_layer)qlayer = new_layer- import auto_round.export.export_to_autoround.qlinear_triton- if sym and isinstance(QuantLinear, (auto_round.export.export_to_autoround.qlinear_triton.QuantLinear,- auto_round_extension.triton.qlinear_tritonv2.QuantLinear,+ if sym and isinstance(QuantLinear, (auto_round_extension.triton.qlinear_tritonv2.QuantLinear,auto_round_extension.torch.qlinear_torch.QuantLinear)):zp = int(zp.flatten()[0])@@ -233,8 +230,6 @@ def pack_layer(layer_name, model, backend):set_module(model, layer_name, qlayer)--def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):"""Saves a quantized model in the auto-round format.@@ -279,7 +274,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exblock_name_to_quantize = quantization_config["block_name_to_quantize"]if isinstance(block_name_to_quantize, str): \block_name_to_quantize = block_name_to_quantize.split(",")- elif isinstance(block_name_to_quantize,list):+ elif isinstance(block_name_to_quantize, list):for i in range(len(block_name_to_quantize)):block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')@@ -381,5 +376,3 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)--diff --git a/auto_round/export/export_to_autoround/qlinear_triton.py b/auto_round/export/export_to_autoround/qlinear_triton.pydeleted file mode 100644index ba2da85e3..000000000--- a/auto_round/export/export_to_autoround/qlinear_triton.py+++ /dev/null@@ -1,137 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import math--import numpy as np-import torch-import torch.nn as nn-import transformers--class TritonModuleMixin:- @classmethod- def warmup(cls, model, transpose=False, seqlen=2048):- pass---class QuantLinear(nn.Module, TritonModuleMixin):- QUANT_TYPE = "triton"-- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):- super().__init__()- if bits not in [2, 4, 8]:- raise NotImplementedError("Only 2,4,8 bits are supported.")- if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError("in_feature and out_feature must be divisible by 32.")- self.infeatures = infeatures- self.outfeatures = outfeatures- self.bits = bits- self.group_size = group_size if group_size != -1 else infeatures- self.maxq = 2 ** self.bits - 1-- self.register_buffer(- "qweight",- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),- )- self.register_buffer(- "qzeros",- torch.zeros(- (- math.ceil(infeatures / self.group_size),- outfeatures // 32 * self.bits,- ),- dtype=torch.int32,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (math.ceil(infeatures / self.group_size), outfeatures),- dtype=torch.float16,- ),- )-- if bias:- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))- else:- self.bias = None-- self.trainable = trainable-- def post_init(self):- pass-- def pack(self, linear, scales, zeros, g_idx=None):- scales_t = scales.t().contiguous()- if linear.bias is not None:- self.bias = linear.bias.clone().half()- self.scales = scales_t.clone().half()- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"-- W = linear.weight.data.to(device).clone()- if isinstance(linear, nn.Conv2d):- W = W.flatten(1)- if isinstance(linear, transformers.pytorch_utils.Conv1D):- W = W.t()-- repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)- if isinstance(zeros, torch.Tensor):- repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(- torch.int32)- else:- repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(- torch.int32)-- del repeat_scales- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits- intweight = intweight << order_map- intweight = torch.sum(intweight, dim=-1)-- intweight = intweight.t().contiguous().to(torch.int32)- self.qweight = intweight.to("cpu")-- if isinstance(zeros, torch.Tensor):- zeros = zeros.t().contiguous()- zeros -= 1- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)- i = 0- col = 0- while col < qzeros.shape[1]:- for j in range(i, i + (32 // self.bits)):- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))- i += 32 // self.bits- col += 1-- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)- else:- zeros -= 1- shape = scales_t.shape- value = 0- for j in range(0, (32 // self.bits)):- value |= zeros << (self.bits * j)- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)---__all__ = ["QuantLinear"]\ No newline at end of filediff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex a44b1fedd..1e6a565b5 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -37,7 +37,7 @@ supported_devices = ("cpu", "hpu", "xpu", "cuda")def flatten_list(nested_list):flattened = []for item in nested_list:- if isinstance(item, list):+ if isinstance(item, (list, tuple)):flattened.extend(flatten_list(item))else:flattened.append(item)@@ -231,7 +231,7 @@ def get_layer_config(model, quantization_config):if to_quant_block_names is None:to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)if isinstance(to_quant_block_names, (list, tuple)):- quant_block_list = to_quant_block_names+ quant_block_list = flatten_list(to_quant_block_names)elif isinstance(to_quant_block_names, str):# Generate quant block names based on the given layer namesquant_block_list = to_quant_block_names.split(",")@@ -243,14 +243,12 @@ def get_layer_config(model, quantization_config):for i in range(len(quant_block_list)):quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip('.')--# Get layer names that will be quantizedlayer_names = []- for n,m in model.named_modules():- if not isinstance(m,SUPPORTED_LAYER_TYPES):+ for n, m in model.named_modules():+ if not isinstance(m, SUPPORTED_LAYER_TYPES):continue- if check_start_with_block_name(n,quant_block_list):+ if check_start_with_block_name(n, quant_block_list):layer_names.append(n)# Load extra configuration if available@@ -263,7 +261,6 @@ def get_layer_config(model, quantization_config):for layer_name in layer_names:if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers-# Process AWQ format: exclude specified modules from quantizationextra_config = skip_not_convert_modules(model, quantization_config, layer_names, extra_config)@@ -555,7 +552,6 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):BackendInfos[orig_backend].packing_format)if best_backend is not None and best_backend not in used_backends:requirements = BackendInfos[best_backend].requirements- process_requirement(requirements, target_device,"warning")+ process_requirement(requirements, target_device, "warning")return model, used_backends-diff --git a/test/test_cuda/test_exllamav2_backend.py b/test/test_cuda/test_exllamav2_backend.pyindex 151cf690a..2a6539306 100644--- a/test/test_cuda/test_exllamav2_backend.py+++ b/test/test_cuda/test_exllamav2_backend.py@@ -77,10 +77,10 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):sym=sym,iters=1,seqlen=2,- dataset=self.llm_dataloader,+ dataset=self.llm_dataloader)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")quantization_config = AutoRoundConfig(backend="gptqmodel:exllamav2")model = AutoModelForCausalLM.from_pretrained(diff --git a/test/test_cuda/test_triton_backend.py b/test/test_cuda/test_triton_backend.pyindex ddb24eca4..a0d69322f 100644--- a/test/test_cuda/test_triton_backend.py+++ b/test/test_cuda/test_triton_backend.py@@ -78,7 +78,7 @@ class TestAutoRoundTritonBackend(unittest.TestCase):dataset=self.llm_dataloader,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path)+ autoround.quantize_and_save(output_dir=quantized_model_path,format="auto_round:gptqmodel")quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(