Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

FP8 WOQ export (#617)

Wenhua Chengefff6ca0bac9
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 0341fc2b2..6cbb50302 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -402,10 +402,10 @@ class AutoRound(object):
raise ValueError("bits must be positive")
if self.act_bits <= 0:
raise ValueError("act_bits must be positive")
- if not (self.group_size == -1 or self.group_size >= 1):
- raise ValueError("group_size must be -1 (per channel) or a positive integer")
- if not (self.act_group_size == -1 or self.act_group_size >= 1):
- raise ValueError("act_group_size must be -1 (per channel) or a positive integer")
+ if not (self.group_size == -1 or self.group_size >= 0):
+ raise ValueError("group_size must be -1 (per channel) or 0 (per-tensor) or a postivie integer")
+ if not (self.act_group_size == -1 or self.act_group_size >= 0):
+ raise ValueError("act_group_size must be -1 (per channel) or 0 (per-tensor) or a positive integer")
if self.batch_size <= 0:
raise ValueError("batch_size must be positive")
if self.iters < 0:
@@ -422,14 +422,17 @@ class AutoRound(object):
"activation quantization is an experimental feature with limited support and a complex API. "
"And please save the quantized model to fake format as real deployment is not supported currently")
- if "mx_fp" in self.data_type:
+ if "mx_fp" in self.data_type or "nv_fp" in self.data_type:
logger.warning(
"please save the quantized model to fake format "
- "as real deployment is not supported for mx_fp datatype currently")
+ "as real deployment is not supported for mx_fp/nv_fp datatype currently")
if "mx_fp" in self.data_type and self.group_size != 32:
logger.warning("mx_fp should only support group_size of 32 in real deployment")
+ if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size!=16):
+ logger.warning("mx_fp should only support group_size of 16/32 in real deployment")
+
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
if self.batch_size > self.nsamples:
logger.warning(f"reset batch_size to {self.nsamples} as nsamples({self.nsamples})"
@@ -495,6 +498,9 @@ class AutoRound(object):
"You can also try to increase the model_max_length to avoid this issue.")
self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)
+ if self.group_size==0 and "fp8" not in self.data_type:
+ logger.warning("group_size of 0 is not supported for data_type other than fp8 ")
+
def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):
"""Quantizes the model and saves it in the specified format(s).
@@ -567,16 +573,19 @@ class AutoRound(object):
for index in range(len(formats)):
format = formats[index]
if format == "auto_round":
- if self.sym or self.bits == 3:
+ if self.sym or self.bits == 3 and "int" in self.data_type:
format = format.replace('auto_round', 'auto_round:auto_gptq')
formats[index] = format
- if self.bits == 4 and not self.sym:
+ if self.bits == 4 and not self.sym and "int" in self.data_type:
enable_awq = all(
config["bits"] == self.bits or config["bits"] >= 16
for config in self.layer_config.values()
)
if enable_awq:
formats[index] = format.replace("auto_round", "auto_round:auto_awq")
+ if "fp8" in self.data_type:
+ format = format.replace("auto_round", f"auto_round:{self.data_type}")
+ formats[index] = format
# Remove duplicates from formats list
def remove_duplicates(lst):
@@ -2044,14 +2053,6 @@ class AutoRound(object):
logger.warning(
"Support for exporting activation quantization is limited. "
"Please ensure that your configuration is supported.")
- if format in ["gguf:q4_0", "gguf:q4_1"]:
- if self.group_size != 32:
- logger.error(f"{format} need group_size=32, but it is {self.group_size}, cannot export.")
- return
- if format == "gguf:q4_0" and not self.sym:
- logger.warning(f"incorrect format choose, will reset to gguf:q4_1")
- if format == "gguf:q4_1" and self.sym:
- logger.warning(f"incorrect format choose, will reset to gguf:q4_0")
from auto_round.export import EXPORT_FORMAT
backend = format
diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.py
index d92c311ad..91de8b187 100644
--- a/auto_round/data_type/fp8.py
+++ b/auto_round/data_type/fp8.py
@@ -13,7 +13,8 @@
# limitations under the License.
import torch
-from auto_round.data_type.utils import get_gaudi_fp8_ste_func, float8_e4m3fn_ste
+from auto_round.data_type.utils import get_gaudi_fp8_ste_func, float8_e4m3fn_ste, reshape_pad_tensor_by_group_size, \
+ revert_tensor_by_pad, float8_e5m2_ste
from auto_round.data_type.register import register_dtype
@@ -57,8 +58,8 @@ def fp8_dynamic_per_token_sym(tensor, max_scale=1.0, **kwargs):
return qdq_res, scale, None
-@register_dtype("fp8_sym")
-def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, **kwargs):
+@register_dtype(("fp8_sym","fp8","fp8_e4m3"))
+def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):
"""Symmetric quantization using float8 format.
Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
@@ -75,16 +76,56 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, **kwargs):
- Scale tensor used for quantization (torch.Tensor).
- Placeholder for zp (None).
"""
- orig_shape = tensor.shape
info = torch.finfo(torch.float8_e4m3fn)
orig_dtype = tensor.dtype
+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor_max is None: ##dynamic per-token
+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[
+ 0] * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.to(tensor.device) * max_scale
+ else:
+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
+ scale = max_tensor.to(torch.float32) / info.max
+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm
+ scale = torch.clip(scale, min=min_scaling_factor)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ scale = scale.unsqueeze(dim=-1)
+ fp8_res = (tensor / scale+v)
+ fp8_res = torch.clip(fp8_res, info.min, info.max)
+ fp8_res = float8_e4m3fn_ste(fp8_res)
+ qdq_res = fp8_res * scale
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+@register_dtype("fp8_e5m2")
+def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e5m2)
+ orig_dtype = tensor.dtype
+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
if tensor_max is None: ##dynamic per-token
- tensor = tensor.reshape(-1, orig_shape[-1])
max_tensor = torch.max(torch.abs(tensor), dim=-1)[
0] * max_scale
- elif isinstance(tensor_max,torch.Tensor):
- max_tensor = tensor_max.clone().detach().to(tensor.device) * max_scale
+ elif isinstance(tensor_max, torch.Tensor):
+ max_tensor = tensor_max.to(tensor.device) * max_scale
else:
max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale
scale = max_tensor.to(torch.float32) / info.max
@@ -93,14 +134,82 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, **kwargs):
if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
tensor = tensor.to(torch.bfloat16)
scale = scale.unsqueeze(dim=-1)
- fp8_res = (tensor / scale)
+ fp8_res = (tensor / scale+v)
+ fp8_res = torch.clip(fp8_res, info.min, info.max)
+ fp8_res = float8_e5m2_ste(fp8_res)
+ qdq_res = fp8_res * scale
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
+@register_dtype("fp8_unit_scale")
+def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e4m3fn)
+ orig_dtype = tensor.dtype
+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ scale = torch.ones((1), device=tensor.device)
+ fp8_res = (tensor / scale+v)
fp8_res = torch.clip(fp8_res, info.min, info.max)
fp8_res = float8_e4m3fn_ste(fp8_res)
qdq_res = fp8_res * scale
- qdq_res = qdq_res.to(orig_dtype).reshape(orig_shape)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
+ return qdq_res, scale, None
+
+
+@register_dtype("fp8_e5m2_unit_scale")
+def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):
+ """Symmetric quantization using float8 format.
+
+ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.
+
+ Args:
+ tensor (torch.Tensor): Input tensor to quantize.
+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.
+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.
+ **kwargs: Additional arguments for compatibility.
+
+ Returns:
+ tuple:
+ - Quantized and dequantized tensor (torch.Tensor).
+ - Scale tensor used for quantization (torch.Tensor).
+ - Placeholder for zp (None).
+ """
+ info = torch.finfo(torch.float8_e5m2)
+ orig_dtype = tensor.dtype
+ tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16
+ tensor = tensor.to(torch.bfloat16)
+ scale = torch.ones((1), device=tensor.device)
+ fp8_res = (tensor / scale+v)
+ fp8_res = torch.clip(fp8_res, info.min, info.max)
+ fp8_res = float8_e5m2_ste(fp8_res)
+ qdq_res = fp8_res * scale
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_res = qdq_res.to(orig_dtype)
return qdq_res, scale, None
+
@register_dtype("fp8_gaudi3_sym")
def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):
"""Symmetric quantization using float8 format.
diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.py
index 1a3b63289..6f22528ca 100644
--- a/auto_round/data_type/register.py
+++ b/auto_round/data_type/register.py
@@ -16,7 +16,7 @@
QUANT_FUNC_WITH_DTYPE = {}
-def register_dtype(name):
+def register_dtype(names):
"""Class decorator to register a EXPORT subclass to the registry.
Decorator function used before a Pattern subclass.
@@ -30,7 +30,12 @@ def register_dtype(name):
"""
def register(dtype):
- QUANT_FUNC_WITH_DTYPE[name] = dtype
+ if isinstance(names,(tuple,list)):
+ for name in names:
+ QUANT_FUNC_WITH_DTYPE[name] = dtype
+ else:
+ QUANT_FUNC_WITH_DTYPE[names] = dtype
+
return dtype
return register
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 87133e4b5..3a15f9b37 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -37,6 +37,9 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int):
"""
orig_shape = data.shape
pad_len = 0
+ if group_size==0:
+ data = data.reshape(1,-1)
+ return data, orig_shape, pad_len
if len(data.shape) > 2:
data = data.reshape(-1, orig_shape[-1])
if group_size == -1 or data.shape[1] < group_size:
@@ -174,6 +177,23 @@ def float8_e4m3fn_ste(x: torch.Tensor):
return fp8
+def float8_e5m2_ste(x: torch.Tensor):
+ """Straight-Through Estimator (STE) for float8.
+
+ Applies a quantization and dequantization step with float8 precision while maintaining
+ gradient flow using a straight-through estimator.
+
+ Args:
+ x (torch.Tensor): Input tensor.
+
+ Returns:
+ torch.Tensor: Quantized and dequantized tensor using float8 format.
+ """
+ fp8 = (x.to(torch.float8_e5m2).to(x.dtype) - x).detach() + x
+
+ return fp8
+
+
def float8_e4m3fn_hpu_ste(x: torch.Tensor):
"""Straight-Through Estimator (STE) for float8.
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 690533398..c4dd1c59d 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -170,7 +170,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
layer_config = kwargs["layer_config"]
names = list(layer_config.keys())
max_workers = 1
- if not torch.cuda.is_available():
+ if not torch.cuda.is_available() or not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
with ThreadPoolExecutor(max_workers=max_workers) as executor:
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index b40c7fe2f..186403dc2 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -21,12 +21,6 @@ import torch
import torch.nn as nn
import transformers
-try:
- import auto_round_extension.triton.qlinear_tritonv2
-except Exception as error:
- raise ImportError(error)
-import auto_round_extension.torch.qlinear_torch
-import auto_round.export.export_to_autoround.qlinear_triton_act
from auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \
filter_quantization_config, SUPPORTED_FORMATS
import threadpoolctl as tctl
@@ -76,6 +70,7 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
"""
if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:
if act_bits <= 8: ##easily have bug for other configuration, need to refine code later
+ import auto_round.export.export_to_autoround.qlinear_triton_act
return auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinear
from auto_round_extension.torch.qlinear_torch import QuantLinear
return QuantLinear
@@ -112,6 +107,7 @@ def pack_qact_layer(name, model):
w_bf16_to_fp8_scale = layer.w_bf16_to_fp8_scale if hasattr(layer, "w_bf16_to_fp8_scale") else None
scale = layer.scale
zp = layer.zp
+ import auto_round.export.export_to_autoround.qlinear_triton_act
QuantLinear = auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinear
if isinstance(layer, nn.Linear):
@@ -165,6 +161,10 @@ def pack_layer(layer_name, model, backend):
if int(layer.act_bits) <= 8:
return pack_qact_layer(layer_name, model)
+ if "fp8" in backend:
+ from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer
+ return pack_layer(layer_name,model,backend)
+
if not check_to_quantized(layer):
return
@@ -196,8 +196,8 @@ def pack_layer(layer_name, model, backend):
new_layer.device = device
set_module(model, layer_name, new_layer)
qlayer = new_layer
- if sym and isinstance(QuantLinear, (auto_round_extension.triton.qlinear_tritonv2.QuantLinear,
- auto_round_extension.torch.qlinear_torch.QuantLinear)):
+ import auto_round_extension.torch.qlinear_torch
+ if sym and isinstance(QuantLinear, (auto_round_extension.torch.qlinear_torch.QuantLinear)):
zp = int(zp.flatten()[0])
qlayer.to("cpu")
@@ -252,6 +252,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
Raises:
ValueError: If the backend is not supported.
"""
+ if "fp8" in kwargs.get("data_type", None) and kwargs.get("act_bits", 16) >= 16:
+ from auto_round.export.export_to_autoround.export_to_fp8_woq import save_quantized_as_autoround
+ return save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)
##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source
if (kwargs.get("sym") is None or kwargs.get("sym") == True) and ("gptq" not in backend and "awq" not in backend):
@@ -304,7 +307,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
max_workers = 1
- if not torch.cuda.is_available():
+ if not torch.cuda.is_available() or not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
new file mode 100644
index 000000000..598a5a8fe
--- /dev/null
+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
@@ -0,0 +1,259 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import os
+import torch
+import json
+from auto_round.utils import logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \
+ filter_quantization_config, get_module, check_start_with_block_name
+import threadpoolctl as tctl
+import transformers
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
+
+
+def check_neq_config(config, data_type, bits, group_size, sym):
+ """
+ Checks if the provided configuration parameters are not equal to the values in the config dictionary.
+
+ Args:
+ config (dict): A dictionary containing the configuration parameters.
+ data_type (str): The expected data type.
+ bits (int): The expected number of bits.
+ group_size (int): The expected group size.
+ sym (bool): The expected symmetry flag.
+
+ Returns:
+ list: A list of strings indicating which configuration parameters do not match.
+ """
+ expected_config = {"data_type": data_type,
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym
+ }
+ return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]
+
+
+class FP8WOQLinear(torch.nn.Module):
+ def __init__(self, in_features, out_features, weight, weight_scale, bias=None, weight_zp=None):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ self.weight = torch.nn.Parameter(weight, requires_grad=False)
+
+ if bias is not None:
+ self.bias = torch.nn.Parameter(bias, requires_grad=False)
+ else:
+ self.register_parameter("bias", None)
+
+ self.register_buffer('weight_scale', weight_scale.to(torch.bfloat16))
+
+ if weight_zp:
+ self.register_buffer('weight_zp', weight_zp.to(torch.bfloat16))
+
+
+def pack_layer(layer_name, model, data_type, packing_device=None):
+ """
+ Packs a model layer for quantization based on its type and configuration.
+
+ This function retrieves the specified layer from the model, checks its
+ compatibility for quantization, and replaces it with a quantized version
+ if applicable. The quantization process depends on the layer's bit-width,
+ group size, symmetry, and activation bits.
+
+ Args:
+ layer_name (str): The name of the layer to be packed.
+ model (torch.nn.Module): The model containing the layer.
+ backend (str): The backend framework to be used for quantization.
+
+ Returns:
+ None: The function modifies the model in place.
+ """
+ if packing_device is None:
+ packing_device = "cpu"
+ if torch.cuda.is_available():
+ packing_device = "cuda"
+ elif torch.xpu.is_available():
+ packing_device = "xpu"
+ layer = get_module(model, layer_name)
+ if hasattr(layer, "orig_layer"):
+ layer = layer.orig_layer
+
+ if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packed
+ return
+
+ if not check_to_quantized(layer):
+ return
+
+ device = layer.weight.device
+ scale = layer.scale
+ zp = layer.zp
+ weight = layer.weight
+ torch_dtype = torch.float8_e4m3fn
+ if "fp8_e5m2" in data_type:
+ torch_dtype = torch.float8_e5m2
+ info = torch.finfo(torch_dtype)
+ if zp is not None:
+ q_weight = weight.to(packing_device) / scale.to(packing_device) + zp.to(packing_device)
+ else:
+ q_weight = weight.to(packing_device) / scale.to(packing_device)
+ q_weight = torch.clamp(q_weight, info.min, info.max)
+ q_weight = q_weight.to(torch_dtype)
+ if isinstance(layer, torch.nn.Linear):
+ in_features = layer.in_features
+ out_features = layer.out_features
+ # elif isinstance(layer, nn.Conv2d):
+ # in_features = layer.in_channels
+ # out_features = layer.out_channels
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):
+ in_features = layer.weight.shape[0]
+ out_features = layer.weight.shape[1]
+ bias = layer.bias
+ my_linear = FP8WOQLinear(in_features, out_features, q_weight, scale, bias, zp)
+
+ my_linear.to(device)
+ set_module(model, layer_name, my_linear)
+
+
+def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round", **kwargs):
+ model = kwargs["model"]
+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
+ if not inplace:
+ model = copy.deepcopy(model.to("cpu"))
+
+ layer_config = kwargs["layer_config"]
+ quantization_config = kwargs["serialization_dict"]
+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
+ quantization_config["quant_method"] = "auto-round"
+ if "e5m2" in kwargs.get("data_type", "fp8"):
+ quantization_config["fmt"] = "e5m2"
+ else:
+ quantization_config["fmt"] = "e4m3"
+ quantization_config["activation_scheme"] = "dynamic"
+
+ tokenizer = kwargs.get("tokenizer", None)
+ processor = kwargs.get("processor", None)
+ extra_config = {}
+ block_name_to_quantize = quantization_config["block_name_to_quantize"]
+ if isinstance(block_name_to_quantize, str): \
+ block_name_to_quantize = block_name_to_quantize.split(",")
+ elif isinstance(block_name_to_quantize, list):
+ for i in range(len(block_name_to_quantize)):
+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')
+
+ for layer_name in layer_config:
+ if not layer_config[layer_name]["in_blocks"] and layer_config[layer_name][
+ "bits"] <= 8: ##lm head ##TODO fix act and so on
+ extra_config[layer_name] = {}
+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]
+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]
+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]
+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]
+ elif layer_config[layer_name]["in_blocks"] or (
+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):
+ neq_keys = check_neq_config(
+ layer_config[layer_name],
+ data_type=quantization_config["data_type"],
+ bits=quantization_config["bits"],
+ group_size=quantization_config["group_size"],
+ sym=quantization_config["sym"],
+ )
+ if len(neq_keys) > 0:
+ extra_config[layer_name] = {}
+ for key in neq_keys:
+ if layer_config[layer_name][key] is not None:
+ extra_config[layer_name][key] = layer_config[layer_name][key]
+ if len(extra_config) > 0:
+ quantization_config["extra_config"] = extra_config
+ names = list(layer_config.keys())
+ max_workers = 1
+ if not torch.cuda.is_available() or not torch.xpu.is_available():
+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally
+ packing_device = "cpu"
+ if torch.cuda.is_available():
+ packing_device = "cuda"
+ elif torch.xpu.is_available():
+ packing_device = "xpu"
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, model, kwargs.get("data_type", "fp8"), packing_device)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
+ filter_quantization_config(quantization_config)
+ if hasattr(model, "config"):
+ model.config.quantization_config = quantization_config
+ if output_dir is None:
+ return model
+
+ if output_dir is None:
+ model.tokenizer = tokenizer
+ return model
+ if os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+
+ if processor is not None:
+ processor.save_pretrained(output_dir)
+ if quantization_config.get("act_bits", 16) <= 8:
+ dtype = torch.bfloat16
+ elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):
+ dtype = torch.float16 ## awq kernel only supports float16 on cuda
+ else:
+ dtype = None
+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+
+ return model
+
+
+def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True,
+ dtype=None):
+ """Save model state dict and configs.
+
+ Args:
+ model (`nn.Module`):
+ Model to be saved. The model can be wrapped or unwrapped.
+ save_dir (`str`):
+ Directory to which to save. Will be created if it doesn't exist.
+ max_shard_size (`str`, defaults to `"10GB"`):
+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
+ <Tip warning={true}>
+
+ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
+ which will be bigger than `max_shard_size`.
+
+ </Tip>
+ safe_serialization (`bool`, defaults to `True`):
+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ """
+ os.makedirs(save_dir, exist_ok=True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_path = os.path.join(save_dir, "config.json")
+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
+ with open(config_path, 'r') as file:
+ data = json.load(file)
+ data["torch_dtype"] = str(dtype).split(".")[-1]
+ with open(config_path, 'w') as file:
+ json.dump(data, file, indent=2)
+ config_file = "quantization_config.json"
+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
+ json.dump(model.config.quantization_config, f, indent=2)
diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.py
index 12dc79555..59a238c75 100644
--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py
+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py
@@ -180,52 +180,6 @@ class QuantLinear(nn.Module):
@classmethod
def warmup(cls, model, transpose=False, seqlen=2048):
return
- # """
- # Pre-tunes the quantized kernel
- # """
- # from tqdm import tqdm
- #
- # kn_values = {}
- #
- # for _, m in model.named_modules():
- # if not isinstance(m, cls):
- # continue
- #
- # k = m.infeatures
- # n = m.outfeatures
- #
- # if (k, n) not in kn_values:
- # kn_values[(k, n)] = (
- # m.qweight,
- # m.scales,
- # m.qzeros,
- # m.g_idx,
- # m.bits,
- # m.maxq,
- # )
- #
- # logger.info(f"Found {len(kn_values)} unique KN Linear values.")
- # logger.info("Warming up autotune cache ...")
- # with torch.no_grad():
- # for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):
- # m = 2 ** m
- # for (k, n), (
- # qweight,
- # scales,
- # qzeros,
- # g_idx,
- # bits,
- # maxq,
- # ) in kn_values.items():
- # if transpose:
- # a = torch.randn(m, k, dtype=torch.float16, device=model.device)
- # quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
- # a = torch.randn(m, n, dtype=torch.float16, device=model.device)
- # transpose_quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
- # else:
- # a = torch.randn(m, k, dtype=torch.float16, device=model.device)
- # quant_matmul_inference_only_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
- # del kn_values
__all__ = ["QuantLinear"]
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 28b15c53e..8d18020a4 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -31,7 +31,7 @@ from auto_round.utils import (logger, get_module,
extract_block_names_to_str, SUPPORTED_LAYER_TYPES, filter_quantization_config)
import copy
import json
-from .utils import WQLinear_GEMM
+from auto_round.export.export_to_awq.utils import WQLinear_GEMM
from concurrent.futures import ThreadPoolExecutor
import threadpoolctl as tctl
from tqdm import tqdm
@@ -102,7 +102,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
backend = None
max_workers = 1
- if not torch.cuda.is_available():
+ if not torch.cuda.is_available() or not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.py
index 49f195ee9..3aea03e04 100644
--- a/auto_round/inference/auto_quantizer.py
+++ b/auto_round/inference/auto_quantizer.py
@@ -151,9 +151,14 @@ class AutoHfQuantizer:
)
else:
warning_msg = ""
+ if (quantization_config_from_args is None or
+ not hasattr(quantization_config_from_args, "get_loading_attributes")):
+ # If the quantization_config_from_args is None or does not have get_loading_attributes method,
+ # we will not use it to load the model.
+ quantization_config_from_args = None
+ else:
+ loading_attr_dict = quantization_config_from_args.get_loading_attributes()
- loading_attr_dict = quantization_config_from_args.get_loading_attributes() \
- if quantization_config_from_args is not None else None
if isinstance(quantization_config, dict):
if "auto-round" in quantization_config[
"quant_method"] or quantization_config_from_args.__class__.__name__ == "AutoRoundConfig":
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 3007aa0dc..e458f168c 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -192,7 +192,9 @@ def get_scale_shape(weight, group_size):
Returns:
The shape of the scale tensor to be used for quantization.
"""
- if group_size == -1 or weight.shape[1] < group_size:
+ if group_size == 0:
+ return (1)
+ elif group_size == -1 or weight.shape[1] < group_size:
shape = weight.shape[0]
else:
shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index f85a99970..5fdb57f87 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -34,6 +34,8 @@ def reshape_and_pad_tensor(v, group_size=-1):
Returns:
torch.Tensor: The reshaped tensor. If padding is applied, the padded tensor is returned.
"""
+ if group_size == 0:
+ return v.reshape(1, -1)
if group_size == -1 or v.shape[1] < group_size:
return v
if v.shape[1] % group_size == 0:
@@ -249,15 +251,21 @@ class WrapperLinear(torch.nn.Module):
if isinstance(scale, dict):
_set_dict_attr(scale, "scale")
- else:
+ elif scale.numel()>1:
self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
+ else:
+ self.orig_layer.scale = scale.view(-1).to("cpu")
if zp is not None:
if isinstance(zp, dict):
_set_dict_attr(zp, "zp")
- else:
+ elif zp is None:
+ self.orig_layer.zp = None
+ elif zp.numel()>1 :
zp = zp.reshape(shape[0], -1)
- self.orig_layer.zp = zp.to("cpu") if zp is not None else None
+ self.orig_layer.zp = zp.to("cpu")
+ else:
+ self.orig_layer.zp = zp.view(-1).to("cpu")
else:
self.orig_layer.zp = None