Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

align formats packing device to API (#795)

Weiweidc81a8786d28
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index c22479ab7..31245c96b 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1419,7 +1419,7 @@ class AutoRound(object):
model_type=model_type,
)
else:
- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0])
+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)
# if self.low_gpu_mem_usage:
# clear_memory()
@@ -2973,7 +2973,9 @@ class AutoRound(object):
model_type=model_type,
)
else:
- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])
+ PACKING_LAYER_WITH_FORMAT[target_backend](
+ tmp_m.tmp_name, self.model, self.formats[0], device=self.device
+ )
pbar.set_description("Quantizing done")
pbar.update(1)
pbar.close()
@@ -3080,6 +3082,7 @@ class AutoRound(object):
backend=backend,
to_quant_block_names=self.to_quant_block_names,
quant_block_list=self.quant_block_list,
+ device=self.device,
**kwargs,
)
return compressed_model
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index b4e210408..c66a893f0 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -66,11 +66,11 @@ BLOCK_PATTERNS = [ ## copy from transformers optimum
]
-def pack_layer(name, model, backend):
+def pack_layer(name, model, backend, device=None):
if name == "lm_head": ##dese not support lm-head
return
layer = get_module(model, name)
-
+ orig_device = layer.weight.device
if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packed
return
@@ -81,8 +81,6 @@ def pack_layer(name, model, backend):
group_size = layer.group_size
sym = layer.sym
- device = layer.weight.device
-
QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)
if isinstance(layer, nn.Linear):
@@ -101,7 +99,7 @@ def pack_layer(name, model, backend):
bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
)
- new_layer.device = device
+ new_layer.device = orig_device
set_module(model, name, new_layer)
qlayer = new_layer
scale = layer.scale
@@ -118,10 +116,10 @@ def pack_layer(name, model, backend):
sig = inspect.signature(qlayer.pack)
param_count = len(sig.parameters)
if param_count == 2:
- qlayer.pack(layer, scale)
+ qlayer.pack(layer, scale, device)
else:
- qlayer.pack(layer, scale, zero, None)
- qlayer.to(device)
+ qlayer.pack(layer, scale, zero, None, device)
+ qlayer.to(orig_device)
if hasattr(layer, "weight"):
layer.weight = None
if hasattr(layer, "bias"):
@@ -136,6 +134,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
quant_block_list = kwargs.get("quant_block_list", get_block_names(model))
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ device = kwargs.get("device", None)
image_processor = kwargs.get("image_processor", None)
if output_dir is not None and os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
@@ -193,14 +192,14 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
def wrapper(name):
pbar.set_description(f"packing {name}")
with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend)
+ pack_layer(name, model, backend, device)
pbar.update(1)
for _ in executor.map(wrapper, names):
pass
if output_dir is None:
return model
-
+ quantization_config["provider"] = "auto-round"
quantization_config["quant_method"] = "gptq"
quantization_config.pop("dataset", None) ## pile-10k is not supported in gptq
quantization_config["desc_act"] = False ## for autogptq API
diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.py
index 5c0c07032..eeab76a96 100644
--- a/auto_round/export/export_to_autogptq/qlinear_triton.py
+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py
@@ -77,17 +77,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):
def post_init(self):
pass
- def pack(self, linear, scales, zeros, g_idx=None):
+ def pack(self, linear, scales, zeros, g_idx=None, device=None):
scales_t = scales.t().contiguous()
self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
if linear.bias is not None:
self.bias = linear.bias.clone().half()
self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 6a6c34c88..20257a16b 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -125,13 +125,13 @@ def pack_qact_layer(name, model):
qlayer.to("cpu")
- qlayer.pack(layer, scale, zp, act_scale, w_bf16_to_fp8_scale)
+ qlayer.pack(layer, scale, zp, act_scale, w_bf16_to_fp8_scale, device)
qlayer.to(device)
-def pack_layer(layer_name, model, backend):
+def pack_layer(layer_name, model, backend, device=None):
"""
- Packs a model layer for quantization based on its type and configuration.
+ Packs a model layer for quantization based on its type and configuration.
This function retrieves the specified layer from the model, checks its
compatibility for quantization, and replaces it with a quantized version
@@ -149,12 +149,12 @@ def pack_layer(layer_name, model, backend):
if is_nv_fp(backend) or is_mx_fp(backend):
from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import pack_layer
- return pack_layer(layer_name, model, backend)
+ return pack_layer(layer_name, model, backend, device)
if backend == "auto_round:fp8":
from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
- return pack_layer(layer_name, model, backend)
+ return pack_layer(layer_name, model, backend, device)
layer = get_module(model, layer_name)
if hasattr(layer, "orig_layer"):
@@ -169,7 +169,7 @@ def pack_layer(layer_name, model, backend):
if not check_to_quantized(layer):
return
- device = layer.weight.device
+ orig_device = layer.weight.device
bits = layer.bits
group_size = layer.group_size
sym = layer.sym
@@ -194,7 +194,7 @@ def pack_layer(layer_name, model, backend):
new_layer = QuantLinear( ##pylint: disable=E1123
bits, group_size, in_features, out_features, bias=bias, weight_dtype=layer.weight.dtype
)
- new_layer.device = device
+ new_layer.device = orig_device
set_module(model, layer_name, new_layer)
qlayer = new_layer
import auto_round_extension.torch.qlinear_torch
@@ -211,9 +211,9 @@ def pack_layer(layer_name, model, backend):
sig = inspect.signature(qlayer.pack)
param_count = len(sig.parameters)
if param_count == 2:
- qlayer.pack(layer, scale)
+ qlayer.pack(layer, scale, device=device)
else:
- qlayer.pack(layer, scale, zp, None)
+ qlayer.pack(layer, scale, zp, None, device=device)
qlayer.to(device)
else:
scale = scale.to(torch.float32).t().contiguous()
@@ -225,14 +225,9 @@ def pack_layer(layer_name, model, backend):
if bits != 4:
logger.error("AutoAWQ format only supports 4-bits quantization.")
qlayer = QuantLinear.from_linear(
- linear=layer,
- w_bit=bits,
- group_size=group_size,
- init_only=False,
- scales=scale,
- zeros=zp,
+ linear=layer, w_bit=bits, group_size=group_size, init_only=False, scales=scale, zeros=zp, device=device
)
- qlayer.to(device)
+ qlayer.to(orig_device)
set_module(model, layer_name, qlayer)
if hasattr(layer, "weight"):
layer.weight = None
@@ -287,7 +282,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
quantization_config["quant_method"] = "auto-round"
quantization_config["packing_format"] = backend
-
+ device = kwargs.get("device", None)
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
image_processor = kwargs.get("image_processor", None)
@@ -331,7 +326,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
def wrapper(name):
pbar.set_description(f"packing {name}")
with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend)
+ pack_layer(name, model, backend, device)
pbar.update(1)
for _ in executor.map(wrapper, names):
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index af196d7b6..4ca1a0da8 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -26,7 +26,7 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_
from auto_round.export.export_to_autoround.utils import REQUIRED_CONFIG_KEYS, check_neq_config
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
- _get_device,
+ _get_packing_device,
check_start_with_block_name,
check_to_quantized,
filter_quantization_config,
@@ -68,7 +68,7 @@ class FP8WOQLinear(torch.nn.Module):
self.register_buffer("input_scale", input_scale.to(dtype))
-def pack_layer(layer_name, model, data_type, packing_device=None):
+def pack_layer(layer_name, model, data_type, device=None):
"""
Packs a model layer for quantization based on its type and configuration.
@@ -85,8 +85,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):
Returns:
None: The function modifies the model in place.
"""
- if packing_device is None:
- packing_device = _get_device()
+ packing_device = _get_packing_device(device)
layer = get_module(model, layer_name)
if hasattr(layer, "orig_layer"):
layer = layer.orig_layer
@@ -97,7 +96,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):
if not check_to_quantized(layer):
return
- device = layer.weight.device
+ orig_device = layer.weight.device
scale = layer.scale.view(-1)
zp = layer.zp
weight = layer.weight
@@ -139,7 +138,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):
dtype=model.dtype,
)
- my_linear.to(device)
+ my_linear.to(orig_device)
set_module(model, layer_name, my_linear)
@@ -148,7 +147,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]
if not inplace:
model = copy.deepcopy(model.to("cpu"))
-
layer_config = kwargs["layer_config"]
quantization_config = kwargs["serialization_dict"]
quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
@@ -161,6 +159,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ device = kwargs.get("device", None)
image_processor = kwargs.get("image_processor", None)
extra_config = {}
block_name_to_quantize = quantization_config["block_name_to_quantize"]
@@ -196,14 +195,13 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- packing_device = _get_device()
with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
def wrapper(name):
pbar.set_description(f"packing {name}")
with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, kwargs.get("data_type", "fp8"), packing_device)
+ pack_layer(name, model, kwargs.get("data_type", "fp8"), device)
pbar.update(1)
for _ in executor.map(wrapper, names):
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
index 06ef32a02..9de661b8b 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
@@ -27,6 +27,7 @@ from tqdm import tqdm
from auto_round.export.export_to_autoround.utils import REQUIRED_CONFIG_KEYS, check_neq_config
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
+ _get_packing_device,
check_start_with_block_name,
check_to_quantized,
filter_quantization_config,
@@ -47,11 +48,10 @@ __all__ = [
]
-def pack_layer(name, model, backend):
+def pack_layer(name, model, backend, device=None):
if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature
return
layer = get_module(model, name)
-
if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packed
return
@@ -60,6 +60,7 @@ def pack_layer(name, model, backend):
layer = wp_layer.orig_layer
set_module(model, name, layer)
+ orig_device = layer.weight.device
data_type = layer.data_type
act_bits = layer.act_bits
act_data_type = layer.act_data_type
@@ -68,14 +69,13 @@ def pack_layer(name, model, backend):
return
group_size = layer.group_size
sym = layer.sym
- device = layer.weight.device
if is_nv_fp(act_data_type) and act_bits <= 8:
if not getattr(layer, "input_global_scale", None):
assert hasattr(layer, "act_max")
from auto_round.data_type.nvfp import calculate_gparam
- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu") # , model.device
+ input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu")
setattr(layer, "input_global_scale", input_global_scale)
delattr(layer, "act_max")
@@ -106,19 +106,16 @@ def pack_layer(name, model, backend):
act_data_type=act_data_type,
)
- new_layer.device = device
+ new_layer.device = orig_device
set_module(model, name, new_layer)
qlayer = new_layer
scale = layer.scale
global_scale = getattr(layer, "weight_global_scale", None)
input_global_scale = getattr(layer, "input_global_scale", None)
# zero = layer.zp
- # so far can only pack layer on CPU
- qlayer.to("cpu")
- layer, scale = layer.to("cpu"), scale.to("cpu")
- qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)
+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale, device=device)
## no zeros to handle, as mxfp not support asym quantization
- qlayer.to(device)
+ qlayer.to(orig_device)
def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
@@ -144,6 +141,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
ValueError: If the backend is not supported.
"""
model = kwargs["model"]
+ device = kwargs.get("device", None)
backend = kwargs.get("backend", None)
bits = kwargs.get("bits", None)
data_type = kwargs.get("data_type", None)
@@ -222,7 +220,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
def wrapper(name):
pbar.set_description(f"packing {name}")
with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend)
+ pack_layer(name, model, backend, device)
pbar.update(1)
for _ in executor.map(wrapper, names):
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
index cff38a30a..57702aea7 100644
--- a/auto_round/export/export_to_autoround/qlinear_fp.py
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -38,7 +38,7 @@ import transformers
from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL
from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
-from auto_round.utils import _get_device, is_mx_fp, is_nv_fp
+from auto_round.utils import _get_packing_device, is_mx_fp, is_nv_fp
# from auto_round.utils import get_weight_compress_dtype
logger = getLogger(__name__)
@@ -140,8 +140,8 @@ class QuantLinear(nn.Module):
def post_init(self):
pass
- def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None):
- device = _get_device()
+ def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None, device=None):
+ device = _get_packing_device(device)
if getattr(linear, "bias", None) is not None:
self.bias = linear.bias.detach().to(torch.float16)
diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.py
index ed06024ec..06d257cd3 100644
--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py
+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py
@@ -41,7 +41,7 @@ import torch
import torch.nn as nn
import transformers
-from auto_round.utils import _get_device
+from auto_round.utils import _get_packing_device
logger = getLogger(__name__)
@@ -118,8 +118,8 @@ class QuantLinear(nn.Module):
def post_init(self):
pass
- def pack(self, linear, scales, zeros, act_scales, w_bf16_to_fp8_scale, g_idx=None):
- device = _get_device()
+ def pack(self, linear, scales, zeros, act_scales, w_bf16_to_fp8_scale, g_idx=None, device=None):
+ device = _get_packing_device(device)
scales_t = scales.t().contiguous()
self.act_scales.data.copy_(act_scales.squeeze().clone())
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 9d50a2a4e..bd9b8773b 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -43,7 +43,7 @@ from auto_round.utils import (
)
-def pack_layer(name, model, backend):
+def pack_layer(name, model, backend, device=None):
if name == "lm_head": ##dese not support lm-head
return
layer = get_module(model, name)
@@ -72,6 +72,7 @@ def pack_layer(name, model, backend):
init_only=False,
scales=scale,
zeros=zp,
+ device=device,
)
set_module(model, name, q_linear)
if hasattr(layer, "weight"):
@@ -85,6 +86,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
model = kwargs["model"]
layer_config = kwargs["layer_config"]
to_quant_block_names = kwargs.get("to_quant_block_names", None)
+ device = kwargs.get("device", None)
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
image_processor = kwargs.get("image_processor", None)
@@ -124,7 +126,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
def wrapper(name):
pbar.set_description(f"packing {name}")
with tctl.threadpool_limits(limits=1):
- pack_layer(name, compressed_model, backend)
+ pack_layer(name, compressed_model, backend, device)
pbar.update(1)
for _ in executor.map(wrapper, names):
@@ -141,11 +143,10 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
for key in layer_config.keys():
if not check_to_quantized(layer_config[key]) and not any(name in key for name in modules_to_not_convert):
modules_to_not_convert.append(key)
-
+ quantization_config["provider"] = "auto-round"
quantization_config["quant_method"] = "awq"
quantization_config["zero_point"] = not quantization_config["sym"]
quantization_config["version"] = "gemm"
-
quantization_config["modules_to_not_convert"] = modules_to_not_convert
##check module quantized in block, this may have bug for mixed precision quantization
filter_quantization_config(quantization_config)
diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.py
index a570b2a9c..bb0cf3921 100644
--- a/auto_round/export/export_to_awq/utils.py
+++ b/auto_round/export/export_to_awq/utils.py
@@ -39,6 +39,8 @@ import torch
import torch.nn as nn
from torch.autograd import Function
+from auto_round.utils import _get_packing_device
+
def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):
shifts = torch.arange(0, 32, bits, device=qzeros.device)
@@ -190,7 +192,8 @@ class WQLinear_GEMM(nn.Module):
self.bias = None
@classmethod
- def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None):
+ def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None, device=None):
+ device = _get_packing_device(device)
awq_linear = cls(
w_bit,
group_size,
@@ -211,11 +214,6 @@ class WQLinear_GEMM(nn.Module):
awq_linear.bias = linear.bias.clone().half()
pack_num = 32 // awq_linear.w_bit
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
repeat_size = group_size if group_size != -1 else linear.in_features
repeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)
if isinstance(zeros, torch.Tensor):
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 6cefa11a5..c8fdbf5f3 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -50,7 +50,7 @@ from transformers import AutoConfig
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.export.export_to_gguf.packing import ggml_quant
-from auto_round.utils import LazyImport, _is_fp8_model, clean_module_parameter, get_module, logger
+from auto_round.utils import LazyImport, _get_packing_device, _is_fp8_model, clean_module_parameter, get_module, logger
gguf = LazyImport("gguf")
@@ -179,12 +179,7 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):
- if torch.cuda.is_available():
- device = "cuda"
- elif torch.xpu.is_available():
- device = "xpu"
- else:
- device = "cpu"
+ device = _get_packing_device()
data_torch = data_torch.to(torch.float32)
scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale
zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp
@@ -209,12 +204,7 @@ def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=
def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid):
suffix = ".weight"
- if torch.cuda.is_available():
- device = "cuda"
- elif torch.xpu.is_available():
- device = "xpu"
- else:
- device = "cpu"
+ device = _get_packing_device()
if suffix in name:
layer_name = name[: -len(suffix)]
module = get_module(cls.model, layer_name)
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 139d4423d..8828df49e 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -38,7 +38,7 @@ def recover_qweight(qdq_weight, scale):
return (qdq_weight / scale).to(torch.int8)
-def pack_layer(layer_name, model, backend):
+def pack_layer(layer_name, model, backend, device=None):
"""
Packs a model layer for quantization based on its type and configuration.
@@ -58,7 +58,7 @@ def pack_layer(layer_name, model, backend):
if is_nv_fp(backend) or is_mx_fp(backend):
from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer
- return pack_layer(layer_name, model, backend)
+ return pack_layer(layer_name, model, backend, device)
## passed as no other llm_compressor format is supported yet
logger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index 0b989f65d..b5e4a9248 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -48,11 +48,10 @@ __all__ = [
]
-def pack_layer(name, model, backend):
+def pack_layer(name, model, backend, device=None):
if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature
return
layer = get_module(model, name)
-
if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packed
return
@@ -61,6 +60,7 @@ def pack_layer(name, model, backend):
layer = wp_layer.orig_layer
set_module(model, name, layer)
+ orig_device = layer.weight.device
data_type = layer.data_type
act_bits = layer.act_bits
act_data_type = layer.act_data_type
@@ -69,7 +69,6 @@ def pack_layer(name, model, backend):
return
group_size = layer.group_size
sym = layer.sym
- device = layer.weight.device
if is_nv_fp(act_data_type) and act_bits <= 8:
if not getattr(layer, "input_global_scale", None):
@@ -107,19 +106,16 @@ def pack_layer(name, model, backend):
act_data_type=act_data_type,
)
- new_layer.device = device
+ new_layer.device = orig_device
set_module(model, name, new_layer)
qlayer = new_layer
scale = layer.scale
global_scale = getattr(layer, "weight_global_scale", None)
input_global_scale = getattr(layer, "input_global_scale", None)
# zero = layer.zp
- # so far can only pack layer on CPU
- qlayer.to("cpu")
- layer, scale = layer.to("cpu"), scale.to("cpu")
- qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)
+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale, device=device)
## no zeros to handle, as mxfp not support asym quantization
- qlayer.to(device)
+ qlayer.to(orig_device)
def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
@@ -154,6 +150,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
if not inplace:
model = copy.deepcopy(model.to("cpu"))
layer_config = kwargs["layer_config"]
+ device = kwargs.get("device", None)
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
extra_config = {}
@@ -189,7 +186,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
def wrapper(name):
pbar.set_description(f"packing {name}")
with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend)
+ pack_layer(name, model, backend, device)
pbar.update(1)
for _ in executor.map(wrapper, names):
@@ -211,6 +208,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
setattr(quantization_config, "format", "nvfp4-pack-quantized")
setattr(quantization_config, "ignore", ignore)
quantization_config = quantization_config.to_dict()
+ quantization_config["provider"] = "auto-round"
if is_mx_fp(data_type): # Manually replace some parameters, as compressed-tensor is not support MXFP scheme yet
quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] = bits
quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] = (
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 355909979..d019420a0 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -2649,10 +2649,34 @@ def get_max_vram(ratio: float = 0.9) -> dict:
return max_memory
-def _get_device() -> torch.device:
- """Selects best available device (CUDA > XPU > CPU)."""
- if torch.cuda.is_available():
- return torch.device("cuda:0")
- if hasattr(torch, "xpu") and torch.xpu.is_available():
- return torch.device("xpu:0")
- return torch.device("cpu")
+def _get_packing_device(device: str | torch.device | None = "auto") -> torch.device:
+ """
+ Selects the packing device.
+ - "auto": choose best available (CUDA > XPU > CPU).
+ - str: parsed by torch.device (e.g., "cuda:2", "cpu").
+ - torch.device: returned as-is.
+ - None: treated as "auto".
+
+ Args:
+ device: Target device spec ("auto", "cuda:0", "xpu:0", "cpu", or torch.device).
+
+ Returns:
+ torch.device: The resolved device.
+ """
+ if device is None or (isinstance(device, str) and device.lower() == "auto"):
+ if torch.cuda.is_available():
+ return torch.device("cuda:0")
+ if hasattr(torch, "xpu") and torch.xpu.is_available():
+ return torch.device("xpu:0")
+ return torch.device("cpu")
+
+ if isinstance(device, torch.device):
+ return device
+
+ if isinstance(device, str):
+ try:
+ return torch.device(device)
+ except Exception as e:
+ raise ValueError(f"Invalid device string: {device}") from e
+
+ raise TypeError(f"Unsupported device type: {type(device)} ({device})")
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index 2b3d4fa1c..a7e40277f 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -20,6 +20,8 @@ import torch
import torch.nn as nn
import transformers
+from auto_round.utils import _get_packing_device
+
logger = getLogger(__name__)
@@ -87,16 +89,12 @@ class QuantLinear(nn.Module):
pass
# @torch.compile() ## cpu side has bug
- def pack_248_bits(self, linear, scales, zeros, g_idx=None):
+ def pack_248_bits(self, linear, scales, zeros, g_idx=None, device=None):
+ device = _get_packing_device(device)
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
@@ -150,16 +148,12 @@ class QuantLinear(nn.Module):
self.qzeros = qzeros.cpu()
# @torch.compile()
- def pack_3bits(self, linear, scales, zeros, g_idx=None):
+ def pack_3bits(self, linear, scales, zeros, g_idx=None, device=None):
+ device = _get_packing_device(device)
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
@@ -267,11 +261,11 @@ class QuantLinear(nn.Module):
qzeros[:] = full_row.unsqueeze(0)
self.qzeros = qzeros.cpu()
- def pack(self, linear, scales, zeros, g_idx=None):
+ def pack(self, linear, scales, zeros, g_idx=None, device=None):
if self.bits in [2, 4, 8]:
- return self.pack_248_bits(linear, scales, zeros, g_idx)
+ return self.pack_248_bits(linear, scales, zeros, g_idx, device)
elif self.bits in [3]:
- return self.pack_3bits(linear, scales, zeros, g_idx)
+ return self.pack_3bits(linear, scales, zeros, g_idx, device)
else:
raise ValueError("Only 2,3,4,8 bits are supported.")
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index 1ddbb363e..244159ef0 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -21,6 +21,8 @@ import torch
import torch.nn as nn
import transformers
+from auto_round.utils import _get_packing_device
+
logger = getLogger(__name__)
@@ -88,16 +90,12 @@ class QuantLinear(nn.Module):
pass
# @torch.compile()
- def pack_248_bits(self, linear, scales, zeros, g_idx=None):
+ def pack_248_bits(self, linear, scales, zeros, g_idx=None, device=None):
+ device = _get_packing_device(device)
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
@@ -150,16 +148,12 @@ class QuantLinear(nn.Module):
qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * value
self.qzeros = qzeros.cpu()
- def pack_3bits(self, linear, scales, zeros, g_idx=None):
+ def pack_3bits(self, linear, scales, zeros, g_idx=None, device=None):
+ device = _get_packing_device(device)
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
@@ -268,11 +262,11 @@ class QuantLinear(nn.Module):
qzeros[:] = full_row.unsqueeze(0)
self.qzeros = qzeros.cpu()
- def pack(self, linear, scales, zeros, g_idx=None):
+ def pack(self, linear, scales, zeros, g_idx=None, device=None):
if self.bits in [2, 4, 8]:
- return self.pack_248_bits(linear, scales, zeros, g_idx)
+ return self.pack_248_bits(linear, scales, zeros, g_idx, device)
elif self.bits in [3]:
- return self.pack_3bits(linear, scales, zeros, g_idx)
+ return self.pack_3bits(linear, scales, zeros, g_idx, device)
else:
raise ValueError("Only 2,3,4,8 bits are supported.")
diff --git a/auto_round_extension/triton/qlinear_tritonv2.py b/auto_round_extension/triton/qlinear_tritonv2.py
index 05b6fea15..4cbf706b3 100644
--- a/auto_round_extension/triton/qlinear_tritonv2.py
+++ b/auto_round_extension/triton/qlinear_tritonv2.py
@@ -20,6 +20,7 @@ import torch
import torch.nn as nn
import transformers
+from auto_round.utils import _get_packing_device
from auto_round_extension.triton.triton_utils.mixin import TritonModuleMixin
logger = getLogger(__name__)
@@ -102,16 +103,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):
def post_init(self):
pass
- def pack(self, linear, scales, zeros, g_idx=None):
+ def pack(self, linear, scales, zeros, g_idx=None, device=None):
+ device = _get_packing_device(device)
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):