Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix cuda ut (#592)

Wenhua Cheng4c3dc07346a6
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b8ee17dd0..03acf72cc 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -167,6 +167,8 @@ class AutoRound(object):
model_kwargs: dict = None,
**kwargs,
):
+ if kwargs:
+ logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
self.quantized = False
self.model_orig_dtype = model.dtype
self.seed = seed
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 42fc63e6d..b40c7fe2f 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -20,6 +20,7 @@ import os
import torch
import torch.nn as nn
import transformers
+
try:
import auto_round_extension.triton.qlinear_tritonv2
except Exception as error:
@@ -27,12 +28,12 @@ except Exception as error:
import auto_round_extension.torch.qlinear_torch
import auto_round.export.export_to_autoround.qlinear_triton_act
from auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \
- filter_quantization_config
+ filter_quantization_config, SUPPORTED_FORMATS
import threadpoolctl as tctl
import inspect
from tqdm import tqdm
from concurrent.futures import ThreadPoolExecutor
-from auto_round.utils import get_autogptq_packing_qlinear,check_start_with_block_name
+from auto_round.utils import get_autogptq_packing_qlinear, check_start_with_block_name
def check_neq_config(config, data_type, bits, group_size, sym):
@@ -78,22 +79,20 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
return auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinear
from auto_round_extension.torch.qlinear_torch import QuantLinear
return QuantLinear
- elif "auto_round" in backend and "gptq" in backend:
+ elif "gptqmodel" in backend:
+ from auto_round_extension.torch.qlinear_torch import QuantLinear
+ return QuantLinear
+ elif "auto_round" in backend and "gptq" in backend and "gptqmodel" not in backend:
from auto_round_extension.torch.qlinear_torch_zp import QuantLinear
return QuantLinear
elif "awq" in backend:
from ..export_to_awq.utils import WQLinear_GEMM
return WQLinear_GEMM
- elif "gptqmodel" in backend:
- from auto_round_extension.torch.qlinear_torch import QuantLinear
- return QuantLinear
elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
else:
raise ValueError(
- f"Unsupported backend: '{backend}'. "
- "Only 'auto_round', 'auto_round:auto_awq', 'auto_round:auto_gptq', 'awq', and 'gptq' are supported."
- )
+ f"unsupported backend: '{backend}'. Supported backends are: {', '.join(SUPPORTED_FORMATS)}")
def pack_qact_layer(name, model):
@@ -197,9 +196,7 @@ def pack_layer(layer_name, model, backend):
new_layer.device = device
set_module(model, layer_name, new_layer)
qlayer = new_layer
- import auto_round.export.export_to_autoround.qlinear_triton
- if sym and isinstance(QuantLinear, (auto_round.export.export_to_autoround.qlinear_triton.QuantLinear,
- auto_round_extension.triton.qlinear_tritonv2.QuantLinear,
+ if sym and isinstance(QuantLinear, (auto_round_extension.triton.qlinear_tritonv2.QuantLinear,
auto_round_extension.torch.qlinear_torch.QuantLinear)):
zp = int(zp.flatten()[0])
@@ -233,8 +230,6 @@ def pack_layer(layer_name, model, backend):
set_module(model, layer_name, qlayer)
-
-
def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):
"""
Saves a quantized model in the auto-round format.
@@ -279,7 +274,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
block_name_to_quantize = quantization_config["block_name_to_quantize"]
if isinstance(block_name_to_quantize, str): \
block_name_to_quantize = block_name_to_quantize.split(",")
- elif isinstance(block_name_to_quantize,list):
+ elif isinstance(block_name_to_quantize, list):
for i in range(len(block_name_to_quantize)):
block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')
@@ -381,5 +376,3 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
-
-
diff --git a/auto_round/export/export_to_autoround/qlinear_triton.py b/auto_round/export/export_to_autoround/qlinear_triton.py
deleted file mode 100644
index ba2da85e3..000000000
--- a/auto_round/export/export_to_autoround/qlinear_triton.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import math
-
-import numpy as np
-import torch
-import torch.nn as nn
-import transformers
-
-class TritonModuleMixin:
- @classmethod
- def warmup(cls, model, transpose=False, seqlen=2048):
- pass
-
-
-class QuantLinear(nn.Module, TritonModuleMixin):
- QUANT_TYPE = "triton"
-
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
- super().__init__()
- if bits not in [2, 4, 8]:
- raise NotImplementedError("Only 2,4,8 bits are supported.")
- if infeatures % 32 != 0 or outfeatures % 32 != 0:
- raise NotImplementedError("in_feature and out_feature must be divisible by 32.")
- self.infeatures = infeatures
- self.outfeatures = outfeatures
- self.bits = bits
- self.group_size = group_size if group_size != -1 else infeatures
- self.maxq = 2 ** self.bits - 1
-
- self.register_buffer(
- "qweight",
- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
- )
- self.register_buffer(
- "qzeros",
- torch.zeros(
- (
- math.ceil(infeatures / self.group_size),
- outfeatures // 32 * self.bits,
- ),
- dtype=torch.int32,
- ),
- )
- self.register_buffer(
- "scales",
- torch.zeros(
- (math.ceil(infeatures / self.group_size), outfeatures),
- dtype=torch.float16,
- ),
- )
-
- if bias:
- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
- else:
- self.bias = None
-
- self.trainable = trainable
-
- def post_init(self):
- pass
-
- def pack(self, linear, scales, zeros, g_idx=None):
- scales_t = scales.t().contiguous()
- if linear.bias is not None:
- self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
-
- W = linear.weight.data.to(device).clone()
- if isinstance(linear, nn.Conv2d):
- W = W.flatten(1)
- if isinstance(linear, transformers.pytorch_utils.Conv1D):
- W = W.t()
-
- repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
- if isinstance(zeros, torch.Tensor):
- repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(
- torch.int32)
- else:
- repeat_zeros = zeros
- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(
- torch.int32)
-
- del repeat_scales
- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
- intweight = intweight << order_map
- intweight = torch.sum(intweight, dim=-1)
-
- intweight = intweight.t().contiguous().to(torch.int32)
- self.qweight = intweight.to("cpu")
-
- if isinstance(zeros, torch.Tensor):
- zeros = zeros.t().contiguous()
- zeros -= 1
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
- i = 0
- col = 0
- while col < qzeros.shape[1]:
- for j in range(i, i + (32 // self.bits)):
- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
- i += 32 // self.bits
- col += 1
-
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
- else:
- zeros -= 1
- shape = scales_t.shape
- value = 0
- for j in range(0, (32 // self.bits)):
- value |= zeros << (self.bits * j)
- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
-
-
-__all__ = ["QuantLinear"]
\ No newline at end of file
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index a44b1fedd..1e6a565b5 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -37,7 +37,7 @@ supported_devices = ("cpu", "hpu", "xpu", "cuda")
def flatten_list(nested_list):
flattened = []
for item in nested_list:
- if isinstance(item, list):
+ if isinstance(item, (list, tuple)):
flattened.extend(flatten_list(item))
else:
flattened.append(item)
@@ -231,7 +231,7 @@ def get_layer_config(model, quantization_config):
if to_quant_block_names is None:
to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)
if isinstance(to_quant_block_names, (list, tuple)):
- quant_block_list = to_quant_block_names
+ quant_block_list = flatten_list(to_quant_block_names)
elif isinstance(to_quant_block_names, str):
# Generate quant block names based on the given layer names
quant_block_list = to_quant_block_names.split(",")
@@ -243,14 +243,12 @@ def get_layer_config(model, quantization_config):
for i in range(len(quant_block_list)):
quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip('.')
-
-
# Get layer names that will be quantized
layer_names = []
- for n,m in model.named_modules():
- if not isinstance(m,SUPPORTED_LAYER_TYPES):
+ for n, m in model.named_modules():
+ if not isinstance(m, SUPPORTED_LAYER_TYPES):
continue
- if check_start_with_block_name(n,quant_block_list):
+ if check_start_with_block_name(n, quant_block_list):
layer_names.append(n)
# Load extra configuration if available
@@ -263,7 +261,6 @@ def get_layer_config(model, quantization_config):
for layer_name in layer_names:
if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):
extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers
-
# Process AWQ format: exclude specified modules from quantization
extra_config = skip_not_convert_modules(model, quantization_config, layer_names, extra_config)
@@ -555,7 +552,6 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
BackendInfos[orig_backend].packing_format)
if best_backend is not None and best_backend not in used_backends:
requirements = BackendInfos[best_backend].requirements
- process_requirement(requirements, target_device,"warning")
+ process_requirement(requirements, target_device, "warning")
return model, used_backends
-
diff --git a/test/test_cuda/test_exllamav2_backend.py b/test/test_cuda/test_exllamav2_backend.py
index 151cf690a..2a6539306 100644
--- a/test/test_cuda/test_exllamav2_backend.py
+++ b/test/test_cuda/test_exllamav2_backend.py
@@ -77,10 +77,10 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):
sym=sym,
iters=1,
seqlen=2,
- dataset=self.llm_dataloader,
+ dataset=self.llm_dataloader
)
quantized_model_path = self.save_folder
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")
quantization_config = AutoRoundConfig(backend="gptqmodel:exllamav2")
model = AutoModelForCausalLM.from_pretrained(
diff --git a/test/test_cuda/test_triton_backend.py b/test/test_cuda/test_triton_backend.py
index ddb24eca4..a0d69322f 100644
--- a/test/test_cuda/test_triton_backend.py
+++ b/test/test_cuda/test_triton_backend.py
@@ -78,7 +78,7 @@ class TestAutoRoundTritonBackend(unittest.TestCase):
dataset=self.llm_dataloader,
)
quantized_model_path = self.save_folder
- autoround.quantize_and_save(output_dir=quantized_model_path)
+ autoround.quantize_and_save(output_dir=quantized_model_path,format="auto_round:gptqmodel")
quantization_config = AutoRoundConfig(backend="tritonv2")
model = AutoModelForCausalLM.from_pretrained(