PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
FP8 WOQ export (#617)
Wenhua Chengefff6ca0bac9
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 0341fc2b2..6cbb50302 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -402,10 +402,10 @@ class AutoRound(object):raise ValueError("bits must be positive")if self.act_bits <= 0:raise ValueError("act_bits must be positive")- if not (self.group_size == -1 or self.group_size >= 1):- raise ValueError("group_size must be -1 (per channel) or a positive integer")- if not (self.act_group_size == -1 or self.act_group_size >= 1):- raise ValueError("act_group_size must be -1 (per channel) or a positive integer")+ if not (self.group_size == -1 or self.group_size >= 0):+ raise ValueError("group_size must be -1 (per channel) or 0 (per-tensor) or a postivie integer")+ if not (self.act_group_size == -1 or self.act_group_size >= 0):+ raise ValueError("act_group_size must be -1 (per channel) or 0 (per-tensor) or a positive integer")if self.batch_size <= 0:raise ValueError("batch_size must be positive")if self.iters < 0:@@ -422,14 +422,17 @@ class AutoRound(object):"activation quantization is an experimental feature with limited support and a complex API. ""And please save the quantized model to fake format as real deployment is not supported currently")- if "mx_fp" in self.data_type:+ if "mx_fp" in self.data_type or "nv_fp" in self.data_type:logger.warning("please save the quantized model to fake format "- "as real deployment is not supported for mx_fp datatype currently")+ "as real deployment is not supported for mx_fp/nv_fp datatype currently")if "mx_fp" in self.data_type and self.group_size != 32:logger.warning("mx_fp should only support group_size of 32 in real deployment")+ if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size!=16):+ logger.warning("mx_fp should only support group_size of 16/32 in real deployment")+if self.nsamples < self.gradient_accumulate_steps * self.batch_size:if self.batch_size > self.nsamples:logger.warning(f"reset batch_size to {self.nsamples} as nsamples({self.nsamples})"@@ -495,6 +498,9 @@ class AutoRound(object):"You can also try to increase the model_max_length to avoid this issue.")self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)+ if self.group_size==0 and "fp8" not in self.data_type:+ logger.warning("group_size of 0 is not supported for data_type other than fp8 ")+def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):"""Quantizes the model and saves it in the specified format(s).@@ -567,16 +573,19 @@ class AutoRound(object):for index in range(len(formats)):format = formats[index]if format == "auto_round":- if self.sym or self.bits == 3:+ if self.sym or self.bits == 3 and "int" in self.data_type:format = format.replace('auto_round', 'auto_round:auto_gptq')formats[index] = format- if self.bits == 4 and not self.sym:+ if self.bits == 4 and not self.sym and "int" in self.data_type:enable_awq = all(config["bits"] == self.bits or config["bits"] >= 16for config in self.layer_config.values())if enable_awq:formats[index] = format.replace("auto_round", "auto_round:auto_awq")+ if "fp8" in self.data_type:+ format = format.replace("auto_round", f"auto_round:{self.data_type}")+ formats[index] = format# Remove duplicates from formats listdef remove_duplicates(lst):@@ -2044,14 +2053,6 @@ class AutoRound(object):logger.warning("Support for exporting activation quantization is limited. ""Please ensure that your configuration is supported.")- if format in ["gguf:q4_0", "gguf:q4_1"]:- if self.group_size != 32:- logger.error(f"{format} need group_size=32, but it is {self.group_size}, cannot export.")- return- if format == "gguf:q4_0" and not self.sym:- logger.warning(f"incorrect format choose, will reset to gguf:q4_1")- if format == "gguf:q4_1" and self.sym:- logger.warning(f"incorrect format choose, will reset to gguf:q4_0")from auto_round.export import EXPORT_FORMATbackend = formatdiff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex d92c311ad..91de8b187 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -13,7 +13,8 @@# limitations under the License.import torch-from auto_round.data_type.utils import get_gaudi_fp8_ste_func, float8_e4m3fn_ste+from auto_round.data_type.utils import get_gaudi_fp8_ste_func, float8_e4m3fn_ste, reshape_pad_tensor_by_group_size, \+ revert_tensor_by_pad, float8_e5m2_stefrom auto_round.data_type.register import register_dtype@@ -57,8 +58,8 @@ def fp8_dynamic_per_token_sym(tensor, max_scale=1.0, **kwargs):return qdq_res, scale, None-@register_dtype("fp8_sym")-def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, **kwargs):+@register_dtype(("fp8_sym","fp8","fp8_e4m3"))+def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -75,16 +76,56 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, **kwargs):- Scale tensor used for quantization (torch.Tensor).- Placeholder for zp (None)."""- orig_shape = tensor.shapeinfo = torch.finfo(torch.float8_e4m3fn)orig_dtype = tensor.dtype+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor_max is None: ##dynamic per-token+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[+ 0] * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ scale = max_tensor.to(torch.float32) / info.max+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm+ scale = torch.clip(scale, min=min_scaling_factor)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = scale.unsqueeze(dim=-1)+ fp8_res = (tensor / scale+v)+ fp8_res = torch.clip(fp8_res, info.min, info.max)+ fp8_res = float8_e4m3fn_ste(fp8_res)+ qdq_res = fp8_res * scale+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None+@register_dtype("fp8_e5m2")+def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e5m2)+ orig_dtype = tensor.dtype+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if tensor_max is None: ##dynamic per-token- tensor = tensor.reshape(-1, orig_shape[-1])max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale- elif isinstance(tensor_max,torch.Tensor):- max_tensor = tensor_max.clone().detach().to(tensor.device) * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scaleelse:max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scalescale = max_tensor.to(torch.float32) / info.max@@ -93,14 +134,82 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, **kwargs):if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16tensor = tensor.to(torch.bfloat16)scale = scale.unsqueeze(dim=-1)- fp8_res = (tensor / scale)+ fp8_res = (tensor / scale+v)+ fp8_res = torch.clip(fp8_res, info.min, info.max)+ fp8_res = float8_e5m2_ste(fp8_res)+ qdq_res = fp8_res * scale+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None+++@register_dtype("fp8_unit_scale")+def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e4m3fn)+ orig_dtype = tensor.dtype+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = torch.ones((1), device=tensor.device)+ fp8_res = (tensor / scale+v)fp8_res = torch.clip(fp8_res, info.min, info.max)fp8_res = float8_e4m3fn_ste(fp8_res)qdq_res = fp8_res * scale- qdq_res = qdq_res.to(orig_dtype).reshape(orig_shape)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, None+++@register_dtype("fp8_e5m2_unit_scale")+def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ info = torch.finfo(torch.float8_e5m2)+ orig_dtype = tensor.dtype+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = torch.ones((1), device=tensor.device)+ fp8_res = (tensor / scale+v)+ fp8_res = torch.clip(fp8_res, info.min, info.max)+ fp8_res = float8_e5m2_ste(fp8_res)+ qdq_res = fp8_res * scale+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)return qdq_res, scale, None+@register_dtype("fp8_gaudi3_sym")def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):"""Symmetric quantization using float8 format.diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.pyindex 1a3b63289..6f22528ca 100644--- a/auto_round/data_type/register.py+++ b/auto_round/data_type/register.py@@ -16,7 +16,7 @@QUANT_FUNC_WITH_DTYPE = {}-def register_dtype(name):+def register_dtype(names):"""Class decorator to register a EXPORT subclass to the registry.Decorator function used before a Pattern subclass.@@ -30,7 +30,12 @@ def register_dtype(name):"""def register(dtype):- QUANT_FUNC_WITH_DTYPE[name] = dtype+ if isinstance(names,(tuple,list)):+ for name in names:+ QUANT_FUNC_WITH_DTYPE[name] = dtype+ else:+ QUANT_FUNC_WITH_DTYPE[names] = dtype+return dtypereturn registerdiff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 87133e4b5..3a15f9b37 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -37,6 +37,9 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int):"""orig_shape = data.shapepad_len = 0+ if group_size==0:+ data = data.reshape(1,-1)+ return data, orig_shape, pad_lenif len(data.shape) > 2:data = data.reshape(-1, orig_shape[-1])if group_size == -1 or data.shape[1] < group_size:@@ -174,6 +177,23 @@ def float8_e4m3fn_ste(x: torch.Tensor):return fp8+def float8_e5m2_ste(x: torch.Tensor):+ """Straight-Through Estimator (STE) for float8.++ Applies a quantization and dequantization step with float8 precision while maintaining+ gradient flow using a straight-through estimator.++ Args:+ x (torch.Tensor): Input tensor.++ Returns:+ torch.Tensor: Quantized and dequantized tensor using float8 format.+ """+ fp8 = (x.to(torch.float8_e5m2).to(x.dtype) - x).detach() + x++ return fp8++def float8_e4m3fn_hpu_ste(x: torch.Tensor):"""Straight-Through Estimator (STE) for float8.diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 690533398..c4dd1c59d 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -170,7 +170,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exlllayer_config = kwargs["layer_config"]names = list(layer_config.keys())max_workers = 1- if not torch.cuda.is_available():+ if not torch.cuda.is_available() or not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex b40c7fe2f..186403dc2 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -21,12 +21,6 @@ import torchimport torch.nn as nnimport transformers-try:- import auto_round_extension.triton.qlinear_tritonv2-except Exception as error:- raise ImportError(error)-import auto_round_extension.torch.qlinear_torch-import auto_round.export.export_to_autoround.qlinear_triton_actfrom auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \filter_quantization_config, SUPPORTED_FORMATSimport threadpoolctl as tctl@@ -76,6 +70,7 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_"""if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:if act_bits <= 8: ##easily have bug for other configuration, need to refine code later+ import auto_round.export.export_to_autoround.qlinear_triton_actreturn auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinearfrom auto_round_extension.torch.qlinear_torch import QuantLinearreturn QuantLinear@@ -112,6 +107,7 @@ def pack_qact_layer(name, model):w_bf16_to_fp8_scale = layer.w_bf16_to_fp8_scale if hasattr(layer, "w_bf16_to_fp8_scale") else Nonescale = layer.scalezp = layer.zp+ import auto_round.export.export_to_autoround.qlinear_triton_actQuantLinear = auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinearif isinstance(layer, nn.Linear):@@ -165,6 +161,10 @@ def pack_layer(layer_name, model, backend):if int(layer.act_bits) <= 8:return pack_qact_layer(layer_name, model)+ if "fp8" in backend:+ from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer+ return pack_layer(layer_name,model,backend)+if not check_to_quantized(layer):return@@ -196,8 +196,8 @@ def pack_layer(layer_name, model, backend):new_layer.device = deviceset_module(model, layer_name, new_layer)qlayer = new_layer- if sym and isinstance(QuantLinear, (auto_round_extension.triton.qlinear_tritonv2.QuantLinear,- auto_round_extension.torch.qlinear_torch.QuantLinear)):+ import auto_round_extension.torch.qlinear_torch+ if sym and isinstance(QuantLinear, (auto_round_extension.torch.qlinear_torch.QuantLinear)):zp = int(zp.flatten()[0])qlayer.to("cpu")@@ -252,6 +252,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exRaises:ValueError: If the backend is not supported."""+ if "fp8" in kwargs.get("data_type", None) and kwargs.get("act_bits", 16) >= 16:+ from auto_round.export.export_to_autoround.export_to_fp8_woq import save_quantized_as_autoround+ return save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)##if using sym, we change to gptq sym kernel to avoid compiling from auto_round sourceif (kwargs.get("sym") is None or kwargs.get("sym") == True) and ("gptq" not in backend and "awq" not in backend):@@ -304,7 +307,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exquantization_config["extra_config"] = extra_confignames = list(layer_config.keys())max_workers = 1- if not torch.cuda.is_available():+ if not torch.cuda.is_available() or not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pynew file mode 100644index 000000000..598a5a8fe--- /dev/null+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -0,0 +1,259 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import copy+import os+import torch+import json+from auto_round.utils import logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \+ filter_quantization_config, get_module, check_start_with_block_name+import threadpoolctl as tctl+import transformers+from tqdm import tqdm+from concurrent.futures import ThreadPoolExecutor+++def check_neq_config(config, data_type, bits, group_size, sym):+ """+ Checks if the provided configuration parameters are not equal to the values in the config dictionary.++ Args:+ config (dict): A dictionary containing the configuration parameters.+ data_type (str): The expected data type.+ bits (int): The expected number of bits.+ group_size (int): The expected group size.+ sym (bool): The expected symmetry flag.++ Returns:+ list: A list of strings indicating which configuration parameters do not match.+ """+ expected_config = {"data_type": data_type,+ "bits": bits,+ "group_size": group_size,+ "sym": sym+ }+ return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]+++class FP8WOQLinear(torch.nn.Module):+ def __init__(self, in_features, out_features, weight, weight_scale, bias=None, weight_zp=None):+ super().__init__()+ self.in_features = in_features+ self.out_features = out_features+ self.weight = torch.nn.Parameter(weight, requires_grad=False)++ if bias is not None:+ self.bias = torch.nn.Parameter(bias, requires_grad=False)+ else:+ self.register_parameter("bias", None)++ self.register_buffer('weight_scale', weight_scale.to(torch.bfloat16))++ if weight_zp:+ self.register_buffer('weight_zp', weight_zp.to(torch.bfloat16))+++def pack_layer(layer_name, model, data_type, packing_device=None):+ """+ Packs a model layer for quantization based on its type and configuration.++ This function retrieves the specified layer from the model, checks its+ compatibility for quantization, and replaces it with a quantized version+ if applicable. The quantization process depends on the layer's bit-width,+ group size, symmetry, and activation bits.++ Args:+ layer_name (str): The name of the layer to be packed.+ model (torch.nn.Module): The model containing the layer.+ backend (str): The backend framework to be used for quantization.++ Returns:+ None: The function modifies the model in place.+ """+ if packing_device is None:+ packing_device = "cpu"+ if torch.cuda.is_available():+ packing_device = "cuda"+ elif torch.xpu.is_available():+ packing_device = "xpu"+ layer = get_module(model, layer_name)+ if hasattr(layer, "orig_layer"):+ layer = layer.orig_layer++ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packed+ return++ if not check_to_quantized(layer):+ return++ device = layer.weight.device+ scale = layer.scale+ zp = layer.zp+ weight = layer.weight+ torch_dtype = torch.float8_e4m3fn+ if "fp8_e5m2" in data_type:+ torch_dtype = torch.float8_e5m2+ info = torch.finfo(torch_dtype)+ if zp is not None:+ q_weight = weight.to(packing_device) / scale.to(packing_device) + zp.to(packing_device)+ else:+ q_weight = weight.to(packing_device) / scale.to(packing_device)+ q_weight = torch.clamp(q_weight, info.min, info.max)+ q_weight = q_weight.to(torch_dtype)+ if isinstance(layer, torch.nn.Linear):+ in_features = layer.in_features+ out_features = layer.out_features+ # elif isinstance(layer, nn.Conv2d):+ # in_features = layer.in_channels+ # out_features = layer.out_channels+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):+ in_features = layer.weight.shape[0]+ out_features = layer.weight.shape[1]+ bias = layer.bias+ my_linear = FP8WOQLinear(in_features, out_features, q_weight, scale, bias, zp)++ my_linear.to(device)+ set_module(model, layer_name, my_linear)+++def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round", **kwargs):+ model = kwargs["model"]+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]+ if not inplace:+ model = copy.deepcopy(model.to("cpu"))++ layer_config = kwargs["layer_config"]+ quantization_config = kwargs["serialization_dict"]+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)+ quantization_config["quant_method"] = "auto-round"+ if "e5m2" in kwargs.get("data_type", "fp8"):+ quantization_config["fmt"] = "e5m2"+ else:+ quantization_config["fmt"] = "e4m3"+ quantization_config["activation_scheme"] = "dynamic"++ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)+ extra_config = {}+ block_name_to_quantize = quantization_config["block_name_to_quantize"]+ if isinstance(block_name_to_quantize, str): \+ block_name_to_quantize = block_name_to_quantize.split(",")+ elif isinstance(block_name_to_quantize, list):+ for i in range(len(block_name_to_quantize)):+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')++ for layer_name in layer_config:+ if not layer_config[layer_name]["in_blocks"] and layer_config[layer_name][+ "bits"] <= 8: ##lm head ##TODO fix act and so on+ extra_config[layer_name] = {}+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]+ elif layer_config[layer_name]["in_blocks"] or (+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):+ neq_keys = check_neq_config(+ layer_config[layer_name],+ data_type=quantization_config["data_type"],+ bits=quantization_config["bits"],+ group_size=quantization_config["group_size"],+ sym=quantization_config["sym"],+ )+ if len(neq_keys) > 0:+ extra_config[layer_name] = {}+ for key in neq_keys:+ if layer_config[layer_name][key] is not None:+ extra_config[layer_name][key] = layer_config[layer_name][key]+ if len(extra_config) > 0:+ quantization_config["extra_config"] = extra_config+ names = list(layer_config.keys())+ max_workers = 1+ if not torch.cuda.is_available() or not torch.xpu.is_available():+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally+ packing_device = "cpu"+ if torch.cuda.is_available():+ packing_device = "cuda"+ elif torch.xpu.is_available():+ packing_device = "xpu"+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:+ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, model, kwargs.get("data_type", "fp8"), packing_device)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ pass+ filter_quantization_config(quantization_config)+ if hasattr(model, "config"):+ model.config.quantization_config = quantization_config+ if output_dir is None:+ return model++ if output_dir is None:+ model.tokenizer = tokenizer+ return model+ if os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)++ if processor is not None:+ processor.save_pretrained(output_dir)+ if quantization_config.get("act_bits", 16) <= 8:+ dtype = torch.bfloat16+ elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):+ dtype = torch.float16 ## awq kernel only supports float16 on cuda+ else:+ dtype = None+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)++ return model+++def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True,+ dtype=None):+ """Save model state dict and configs.++ Args:+ model (`nn.Module`):+ Model to be saved. The model can be wrapped or unwrapped.+ save_dir (`str`):+ Directory to which to save. Will be created if it doesn't exist.+ max_shard_size (`str`, defaults to `"10GB"`):+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).+ <Tip warning={true}>++ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard+ which will be bigger than `max_shard_size`.++ </Tip>+ safe_serialization (`bool`, defaults to `True`):+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).+ """+ os.makedirs(save_dir, exist_ok=True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ config_path = os.path.join(save_dir, "config.json")+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):+ with open(config_path, 'r') as file:+ data = json.load(file)+ data["torch_dtype"] = str(dtype).split(".")[-1]+ with open(config_path, 'w') as file:+ json.dump(data, file, indent=2)+ config_file = "quantization_config.json"+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:+ json.dump(model.config.quantization_config, f, indent=2)diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.pyindex 12dc79555..59a238c75 100644--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py@@ -180,52 +180,6 @@ class QuantLinear(nn.Module):@classmethoddef warmup(cls, model, transpose=False, seqlen=2048):return- # """- # Pre-tunes the quantized kernel- # """- # from tqdm import tqdm- #- # kn_values = {}- #- # for _, m in model.named_modules():- # if not isinstance(m, cls):- # continue- #- # k = m.infeatures- # n = m.outfeatures- #- # if (k, n) not in kn_values:- # kn_values[(k, n)] = (- # m.qweight,- # m.scales,- # m.qzeros,- # m.g_idx,- # m.bits,- # m.maxq,- # )- #- # logger.info(f"Found {len(kn_values)} unique KN Linear values.")- # logger.info("Warming up autotune cache ...")- # with torch.no_grad():- # for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):- # m = 2 ** m- # for (k, n), (- # qweight,- # scales,- # qzeros,- # g_idx,- # bits,- # maxq,- # ) in kn_values.items():- # if transpose:- # a = torch.randn(m, k, dtype=torch.float16, device=model.device)- # quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)- # a = torch.randn(m, n, dtype=torch.float16, device=model.device)- # transpose_quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)- # else:- # a = torch.randn(m, k, dtype=torch.float16, device=model.device)- # quant_matmul_inference_only_248(a, qweight, scales, qzeros, g_idx, bits, maxq)- # del kn_values__all__ = ["QuantLinear"]diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 28b15c53e..8d18020a4 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -31,7 +31,7 @@ from auto_round.utils import (logger, get_module,extract_block_names_to_str, SUPPORTED_LAYER_TYPES, filter_quantization_config)import copyimport json-from .utils import WQLinear_GEMM+from auto_round.export.export_to_awq.utils import WQLinear_GEMMfrom concurrent.futures import ThreadPoolExecutorimport threadpoolctl as tctlfrom tqdm import tqdm@@ -102,7 +102,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):backend = Nonemax_workers = 1- if not torch.cuda.is_available():+ if not torch.cuda.is_available() or not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pyindex 49f195ee9..3aea03e04 100644--- a/auto_round/inference/auto_quantizer.py+++ b/auto_round/inference/auto_quantizer.py@@ -151,9 +151,14 @@ class AutoHfQuantizer:)else:warning_msg = ""+ if (quantization_config_from_args is None or+ not hasattr(quantization_config_from_args, "get_loading_attributes")):+ # If the quantization_config_from_args is None or does not have get_loading_attributes method,+ # we will not use it to load the model.+ quantization_config_from_args = None+ else:+ loading_attr_dict = quantization_config_from_args.get_loading_attributes()- loading_attr_dict = quantization_config_from_args.get_loading_attributes() \- if quantization_config_from_args is not None else Noneif isinstance(quantization_config, dict):if "auto-round" in quantization_config["quant_method"] or quantization_config_from_args.__class__.__name__ == "AutoRoundConfig":diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 3007aa0dc..e458f168c 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -192,7 +192,9 @@ def get_scale_shape(weight, group_size):Returns:The shape of the scale tensor to be used for quantization."""- if group_size == -1 or weight.shape[1] < group_size:+ if group_size == 0:+ return (1)+ elif group_size == -1 or weight.shape[1] < group_size:shape = weight.shape[0]else:shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex f85a99970..5fdb57f87 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -34,6 +34,8 @@ def reshape_and_pad_tensor(v, group_size=-1):Returns:torch.Tensor: The reshaped tensor. If padding is applied, the padded tensor is returned."""+ if group_size == 0:+ return v.reshape(1, -1)if group_size == -1 or v.shape[1] < group_size:return vif v.shape[1] % group_size == 0:@@ -249,15 +251,21 @@ class WrapperLinear(torch.nn.Module):if isinstance(scale, dict):_set_dict_attr(scale, "scale")- else:+ elif scale.numel()>1:self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")+ else:+ self.orig_layer.scale = scale.view(-1).to("cpu")if zp is not None:if isinstance(zp, dict):_set_dict_attr(zp, "zp")- else:+ elif zp is None:+ self.orig_layer.zp = None+ elif zp.numel()>1 :zp = zp.reshape(shape[0], -1)- self.orig_layer.zp = zp.to("cpu") if zp is not None else None+ self.orig_layer.zp = zp.to("cpu")+ else:+ self.orig_layer.zp = zp.view(-1).to("cpu")else:self.orig_layer.zp = None