PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
align formats packing device to API (#795)
Weiweidc81a8786d28
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c22479ab7..31245c96b 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1419,7 +1419,7 @@ class AutoRound(object):model_type=model_type,)else:- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0])+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)# if self.low_gpu_mem_usage:# clear_memory()@@ -2973,7 +2973,9 @@ class AutoRound(object):model_type=model_type,)else:- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])+ PACKING_LAYER_WITH_FORMAT[target_backend](+ tmp_m.tmp_name, self.model, self.formats[0], device=self.device+ )pbar.set_description("Quantizing done")pbar.update(1)pbar.close()@@ -3080,6 +3082,7 @@ class AutoRound(object):backend=backend,to_quant_block_names=self.to_quant_block_names,quant_block_list=self.quant_block_list,+ device=self.device,**kwargs,)return compressed_modeldiff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex b4e210408..c66a893f0 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -66,11 +66,11 @@ BLOCK_PATTERNS = [ ## copy from transformers optimum]-def pack_layer(name, model, backend):+def pack_layer(name, model, backend, device=None):if name == "lm_head": ##dese not support lm-headreturnlayer = get_module(model, name)-+ orig_device = layer.weight.deviceif not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packedreturn@@ -81,8 +81,6 @@ def pack_layer(name, model, backend):group_size = layer.group_sizesym = layer.sym- device = layer.weight.device-QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)if isinstance(layer, nn.Linear):@@ -101,7 +99,7 @@ def pack_layer(name, model, backend):bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype)- new_layer.device = device+ new_layer.device = orig_deviceset_module(model, name, new_layer)qlayer = new_layerscale = layer.scale@@ -118,10 +116,10 @@ def pack_layer(name, model, backend):sig = inspect.signature(qlayer.pack)param_count = len(sig.parameters)if param_count == 2:- qlayer.pack(layer, scale)+ qlayer.pack(layer, scale, device)else:- qlayer.pack(layer, scale, zero, None)- qlayer.to(device)+ qlayer.pack(layer, scale, zero, None, device)+ qlayer.to(orig_device)if hasattr(layer, "weight"):layer.weight = Noneif hasattr(layer, "bias"):@@ -136,6 +134,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllquant_block_list = kwargs.get("quant_block_list", get_block_names(model))tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ device = kwargs.get("device", None)image_processor = kwargs.get("image_processor", None)if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")@@ -193,14 +192,14 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exlldef wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend)+ pack_layer(name, model, backend, device)pbar.update(1)for _ in executor.map(wrapper, names):passif output_dir is None:return model-+ quantization_config["provider"] = "auto-round"quantization_config["quant_method"] = "gptq"quantization_config.pop("dataset", None) ## pile-10k is not supported in gptqquantization_config["desc_act"] = False ## for autogptq APIdiff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pyindex 5c0c07032..eeab76a96 100644--- a/auto_round/export/export_to_autogptq/qlinear_triton.py+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py@@ -77,17 +77,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):def post_init(self):pass- def pack(self, linear, scales, zeros, g_idx=None):+ def pack(self, linear, scales, zeros, g_idx=None, device=None):scales_t = scales.t().contiguous()self.g_idx = g_idx.clone() if g_idx is not None else self.g_idxif linear.bias is not None:self.bias = linear.bias.clone().half()self.scales = scales_t.clone().half()- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 6a6c34c88..20257a16b 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -125,13 +125,13 @@ def pack_qact_layer(name, model):qlayer.to("cpu")- qlayer.pack(layer, scale, zp, act_scale, w_bf16_to_fp8_scale)+ qlayer.pack(layer, scale, zp, act_scale, w_bf16_to_fp8_scale, device)qlayer.to(device)-def pack_layer(layer_name, model, backend):+def pack_layer(layer_name, model, backend, device=None):"""- Packs a model layer for quantization based on its type and configuration.+ Packs a model layer for quantization based on its type and configuration.This function retrieves the specified layer from the model, checks itscompatibility for quantization, and replaces it with a quantized version@@ -149,12 +149,12 @@ def pack_layer(layer_name, model, backend):if is_nv_fp(backend) or is_mx_fp(backend):from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import pack_layer- return pack_layer(layer_name, model, backend)+ return pack_layer(layer_name, model, backend, device)if backend == "auto_round:fp8":from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer- return pack_layer(layer_name, model, backend)+ return pack_layer(layer_name, model, backend, device)layer = get_module(model, layer_name)if hasattr(layer, "orig_layer"):@@ -169,7 +169,7 @@ def pack_layer(layer_name, model, backend):if not check_to_quantized(layer):return- device = layer.weight.device+ orig_device = layer.weight.devicebits = layer.bitsgroup_size = layer.group_sizesym = layer.sym@@ -194,7 +194,7 @@ def pack_layer(layer_name, model, backend):new_layer = QuantLinear( ##pylint: disable=E1123bits, group_size, in_features, out_features, bias=bias, weight_dtype=layer.weight.dtype)- new_layer.device = device+ new_layer.device = orig_deviceset_module(model, layer_name, new_layer)qlayer = new_layerimport auto_round_extension.torch.qlinear_torch@@ -211,9 +211,9 @@ def pack_layer(layer_name, model, backend):sig = inspect.signature(qlayer.pack)param_count = len(sig.parameters)if param_count == 2:- qlayer.pack(layer, scale)+ qlayer.pack(layer, scale, device=device)else:- qlayer.pack(layer, scale, zp, None)+ qlayer.pack(layer, scale, zp, None, device=device)qlayer.to(device)else:scale = scale.to(torch.float32).t().contiguous()@@ -225,14 +225,9 @@ def pack_layer(layer_name, model, backend):if bits != 4:logger.error("AutoAWQ format only supports 4-bits quantization.")qlayer = QuantLinear.from_linear(- linear=layer,- w_bit=bits,- group_size=group_size,- init_only=False,- scales=scale,- zeros=zp,+ linear=layer, w_bit=bits, group_size=group_size, init_only=False, scales=scale, zeros=zp, device=device)- qlayer.to(device)+ qlayer.to(orig_device)set_module(model, layer_name, qlayer)if hasattr(layer, "weight"):layer.weight = None@@ -287,7 +282,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exquantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)quantization_config["quant_method"] = "auto-round"quantization_config["packing_format"] = backend-+ device = kwargs.get("device", None)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)image_processor = kwargs.get("image_processor", None)@@ -331,7 +326,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exdef wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend)+ pack_layer(name, model, backend, device)pbar.update(1)for _ in executor.map(wrapper, names):diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex af196d7b6..4ca1a0da8 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -26,7 +26,7 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_from auto_round.export.export_to_autoround.utils import REQUIRED_CONFIG_KEYS, check_neq_configfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,- _get_device,+ _get_packing_device,check_start_with_block_name,check_to_quantized,filter_quantization_config,@@ -68,7 +68,7 @@ class FP8WOQLinear(torch.nn.Module):self.register_buffer("input_scale", input_scale.to(dtype))-def pack_layer(layer_name, model, data_type, packing_device=None):+def pack_layer(layer_name, model, data_type, device=None):"""Packs a model layer for quantization based on its type and configuration.@@ -85,8 +85,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):Returns:None: The function modifies the model in place."""- if packing_device is None:- packing_device = _get_device()+ packing_device = _get_packing_device(device)layer = get_module(model, layer_name)if hasattr(layer, "orig_layer"):layer = layer.orig_layer@@ -97,7 +96,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):if not check_to_quantized(layer):return- device = layer.weight.device+ orig_device = layer.weight.devicescale = layer.scale.view(-1)zp = layer.zpweight = layer.weight@@ -139,7 +138,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):dtype=model.dtype,)- my_linear.to(device)+ my_linear.to(orig_device)set_module(model, layer_name, my_linear)@@ -148,7 +147,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]if not inplace:model = copy.deepcopy(model.to("cpu"))-layer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)@@ -161,6 +159,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ device = kwargs.get("device", None)image_processor = kwargs.get("image_processor", None)extra_config = {}block_name_to_quantize = quantization_config["block_name_to_quantize"]@@ -196,14 +195,13 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",max_workers = 1if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionally- packing_device = _get_device()with ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):- pack_layer(name, model, kwargs.get("data_type", "fp8"), packing_device)+ pack_layer(name, model, kwargs.get("data_type", "fp8"), device)pbar.update(1)for _ in executor.map(wrapper, names):diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyindex 06ef32a02..9de661b8b 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py@@ -27,6 +27,7 @@ from tqdm import tqdmfrom auto_round.export.export_to_autoround.utils import REQUIRED_CONFIG_KEYS, check_neq_configfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,+ _get_packing_device,check_start_with_block_name,check_to_quantized,filter_quantization_config,@@ -47,11 +48,10 @@ __all__ = []-def pack_layer(name, model, backend):+def pack_layer(name, model, backend, device=None):if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this featurereturnlayer = get_module(model, name)-if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packedreturn@@ -60,6 +60,7 @@ def pack_layer(name, model, backend):layer = wp_layer.orig_layerset_module(model, name, layer)+ orig_device = layer.weight.devicedata_type = layer.data_typeact_bits = layer.act_bitsact_data_type = layer.act_data_type@@ -68,14 +69,13 @@ def pack_layer(name, model, backend):returngroup_size = layer.group_sizesym = layer.sym- device = layer.weight.deviceif is_nv_fp(act_data_type) and act_bits <= 8:if not getattr(layer, "input_global_scale", None):assert hasattr(layer, "act_max")from auto_round.data_type.nvfp import calculate_gparam- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu") # , model.device+ input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu")setattr(layer, "input_global_scale", input_global_scale)delattr(layer, "act_max")@@ -106,19 +106,16 @@ def pack_layer(name, model, backend):act_data_type=act_data_type,)- new_layer.device = device+ new_layer.device = orig_deviceset_module(model, name, new_layer)qlayer = new_layerscale = layer.scaleglobal_scale = getattr(layer, "weight_global_scale", None)input_global_scale = getattr(layer, "input_global_scale", None)# zero = layer.zp- # so far can only pack layer on CPU- qlayer.to("cpu")- layer, scale = layer.to("cpu"), scale.to("cpu")- qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale, device=device)## no zeros to handle, as mxfp not support asym quantization- qlayer.to(device)+ qlayer.to(orig_device)def save_quantized_as_fp(output_dir, inplace=True, **kwargs):@@ -144,6 +141,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):ValueError: If the backend is not supported."""model = kwargs["model"]+ device = kwargs.get("device", None)backend = kwargs.get("backend", None)bits = kwargs.get("bits", None)data_type = kwargs.get("data_type", None)@@ -222,7 +220,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend)+ pack_layer(name, model, backend, device)pbar.update(1)for _ in executor.map(wrapper, names):diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex cff38a30a..57702aea7 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -38,7 +38,7 @@ import transformersfrom auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMALfrom auto_round.data_type.nvfp import cast_to_fp4, get_reciprocalfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad-from auto_round.utils import _get_device, is_mx_fp, is_nv_fp+from auto_round.utils import _get_packing_device, is_mx_fp, is_nv_fp# from auto_round.utils import get_weight_compress_dtypelogger = getLogger(__name__)@@ -140,8 +140,8 @@ class QuantLinear(nn.Module):def post_init(self):pass- def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None):- device = _get_device()+ def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None, device=None):+ device = _get_packing_device(device)if getattr(linear, "bias", None) is not None:self.bias = linear.bias.detach().to(torch.float16)diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.pyindex ed06024ec..06d257cd3 100644--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py@@ -41,7 +41,7 @@ import torchimport torch.nn as nnimport transformers-from auto_round.utils import _get_device+from auto_round.utils import _get_packing_devicelogger = getLogger(__name__)@@ -118,8 +118,8 @@ class QuantLinear(nn.Module):def post_init(self):pass- def pack(self, linear, scales, zeros, act_scales, w_bf16_to_fp8_scale, g_idx=None):- device = _get_device()+ def pack(self, linear, scales, zeros, act_scales, w_bf16_to_fp8_scale, g_idx=None, device=None):+ device = _get_packing_device(device)scales_t = scales.t().contiguous()self.act_scales.data.copy_(act_scales.squeeze().clone())diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 9d50a2a4e..bd9b8773b 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -43,7 +43,7 @@ from auto_round.utils import ()-def pack_layer(name, model, backend):+def pack_layer(name, model, backend, device=None):if name == "lm_head": ##dese not support lm-headreturnlayer = get_module(model, name)@@ -72,6 +72,7 @@ def pack_layer(name, model, backend):init_only=False,scales=scale,zeros=zp,+ device=device,)set_module(model, name, q_linear)if hasattr(layer, "weight"):@@ -85,6 +86,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):model = kwargs["model"]layer_config = kwargs["layer_config"]to_quant_block_names = kwargs.get("to_quant_block_names", None)+ device = kwargs.get("device", None)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)image_processor = kwargs.get("image_processor", None)@@ -124,7 +126,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):- pack_layer(name, compressed_model, backend)+ pack_layer(name, compressed_model, backend, device)pbar.update(1)for _ in executor.map(wrapper, names):@@ -141,11 +143,10 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):for key in layer_config.keys():if not check_to_quantized(layer_config[key]) and not any(name in key for name in modules_to_not_convert):modules_to_not_convert.append(key)-+ quantization_config["provider"] = "auto-round"quantization_config["quant_method"] = "awq"quantization_config["zero_point"] = not quantization_config["sym"]quantization_config["version"] = "gemm"-quantization_config["modules_to_not_convert"] = modules_to_not_convert##check module quantized in block, this may have bug for mixed precision quantizationfilter_quantization_config(quantization_config)diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex a570b2a9c..bb0cf3921 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -39,6 +39,8 @@ import torchimport torch.nn as nnfrom torch.autograd import Function+from auto_round.utils import _get_packing_device+def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):shifts = torch.arange(0, 32, bits, device=qzeros.device)@@ -190,7 +192,8 @@ class WQLinear_GEMM(nn.Module):self.bias = None@classmethod- def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None):+ def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None, device=None):+ device = _get_packing_device(device)awq_linear = cls(w_bit,group_size,@@ -211,11 +214,6 @@ class WQLinear_GEMM(nn.Module):awq_linear.bias = linear.bias.clone().half()pack_num = 32 // awq_linear.w_bit- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"repeat_size = group_size if group_size != -1 else linear.in_featuresrepeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)if isinstance(zeros, torch.Tensor):diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 6cefa11a5..c8fdbf5f3 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -50,7 +50,7 @@ from transformers import AutoConfigfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.export.export_to_gguf.packing import ggml_quant-from auto_round.utils import LazyImport, _is_fp8_model, clean_module_parameter, get_module, logger+from auto_round.utils import LazyImport, _get_packing_device, _is_fp8_model, clean_module_parameter, get_module, loggergguf = LazyImport("gguf")@@ -179,12 +179,7 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):- if torch.cuda.is_available():- device = "cuda"- elif torch.xpu.is_available():- device = "xpu"- else:- device = "cpu"+ device = _get_packing_device()data_torch = data_torch.to(torch.float32)scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scalezp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp@@ -209,12 +204,7 @@ def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid):suffix = ".weight"- if torch.cuda.is_available():- device = "cuda"- elif torch.xpu.is_available():- device = "xpu"- else:- device = "cpu"+ device = _get_packing_device()if suffix in name:layer_name = name[: -len(suffix)]module = get_module(cls.model, layer_name)diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 139d4423d..8828df49e 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -38,7 +38,7 @@ def recover_qweight(qdq_weight, scale):return (qdq_weight / scale).to(torch.int8)-def pack_layer(layer_name, model, backend):+def pack_layer(layer_name, model, backend, device=None):"""Packs a model layer for quantization based on its type and configuration.@@ -58,7 +58,7 @@ def pack_layer(layer_name, model, backend):if is_nv_fp(backend) or is_mx_fp(backend):from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer- return pack_layer(layer_name, model, backend)+ return pack_layer(layer_name, model, backend, device)## passed as no other llm_compressor format is supported yetlogger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 0b989f65d..b5e4a9248 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -48,11 +48,10 @@ __all__ = []-def pack_layer(name, model, backend):+def pack_layer(name, model, backend, device=None):if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this featurereturnlayer = get_module(model, name)-if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packedreturn@@ -61,6 +60,7 @@ def pack_layer(name, model, backend):layer = wp_layer.orig_layerset_module(model, name, layer)+ orig_device = layer.weight.devicedata_type = layer.data_typeact_bits = layer.act_bitsact_data_type = layer.act_data_type@@ -69,7 +69,6 @@ def pack_layer(name, model, backend):returngroup_size = layer.group_sizesym = layer.sym- device = layer.weight.deviceif is_nv_fp(act_data_type) and act_bits <= 8:if not getattr(layer, "input_global_scale", None):@@ -107,19 +106,16 @@ def pack_layer(name, model, backend):act_data_type=act_data_type,)- new_layer.device = device+ new_layer.device = orig_deviceset_module(model, name, new_layer)qlayer = new_layerscale = layer.scaleglobal_scale = getattr(layer, "weight_global_scale", None)input_global_scale = getattr(layer, "input_global_scale", None)# zero = layer.zp- # so far can only pack layer on CPU- qlayer.to("cpu")- layer, scale = layer.to("cpu"), scale.to("cpu")- qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale, device=device)## no zeros to handle, as mxfp not support asym quantization- qlayer.to(device)+ qlayer.to(orig_device)def save_quantized_as_fp(output_dir, inplace=True, **kwargs):@@ -154,6 +150,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):if not inplace:model = copy.deepcopy(model.to("cpu"))layer_config = kwargs["layer_config"]+ device = kwargs.get("device", None)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)extra_config = {}@@ -189,7 +186,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend)+ pack_layer(name, model, backend, device)pbar.update(1)for _ in executor.map(wrapper, names):@@ -211,6 +208,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):setattr(quantization_config, "format", "nvfp4-pack-quantized")setattr(quantization_config, "ignore", ignore)quantization_config = quantization_config.to_dict()+ quantization_config["provider"] = "auto-round"if is_mx_fp(data_type): # Manually replace some parameters, as compressed-tensor is not support MXFP scheme yetquantization_config["config_groups"]["group_0"]["weights"]["num_bits"] = bitsquantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] = (diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 355909979..d019420a0 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -2649,10 +2649,34 @@ def get_max_vram(ratio: float = 0.9) -> dict:return max_memory-def _get_device() -> torch.device:- """Selects best available device (CUDA > XPU > CPU)."""- if torch.cuda.is_available():- return torch.device("cuda:0")- if hasattr(torch, "xpu") and torch.xpu.is_available():- return torch.device("xpu:0")- return torch.device("cpu")+def _get_packing_device(device: str | torch.device | None = "auto") -> torch.device:+ """+ Selects the packing device.+ - "auto": choose best available (CUDA > XPU > CPU).+ - str: parsed by torch.device (e.g., "cuda:2", "cpu").+ - torch.device: returned as-is.+ - None: treated as "auto".++ Args:+ device: Target device spec ("auto", "cuda:0", "xpu:0", "cpu", or torch.device).++ Returns:+ torch.device: The resolved device.+ """+ if device is None or (isinstance(device, str) and device.lower() == "auto"):+ if torch.cuda.is_available():+ return torch.device("cuda:0")+ if hasattr(torch, "xpu") and torch.xpu.is_available():+ return torch.device("xpu:0")+ return torch.device("cpu")++ if isinstance(device, torch.device):+ return device++ if isinstance(device, str):+ try:+ return torch.device(device)+ except Exception as e:+ raise ValueError(f"Invalid device string: {device}") from e++ raise TypeError(f"Unsupported device type: {type(device)} ({device})")diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 2b3d4fa1c..a7e40277f 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -20,6 +20,8 @@ import torchimport torch.nn as nnimport transformers+from auto_round.utils import _get_packing_device+logger = getLogger(__name__)@@ -87,16 +89,12 @@ class QuantLinear(nn.Module):pass# @torch.compile() ## cpu side has bug- def pack_248_bits(self, linear, scales, zeros, g_idx=None):+ def pack_248_bits(self, linear, scales, zeros, g_idx=None, device=None):+ device = _get_packing_device(device)scales_t = scales.t().contiguous()if linear.bias is not None:self.bias = linear.bias.clone().half()self.scales = scales_t.clone().half()- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):@@ -150,16 +148,12 @@ class QuantLinear(nn.Module):self.qzeros = qzeros.cpu()# @torch.compile()- def pack_3bits(self, linear, scales, zeros, g_idx=None):+ def pack_3bits(self, linear, scales, zeros, g_idx=None, device=None):+ device = _get_packing_device(device)scales_t = scales.t().contiguous()if linear.bias is not None:self.bias = linear.bias.clone().half()self.scales = scales_t.clone().half()- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):@@ -267,11 +261,11 @@ class QuantLinear(nn.Module):qzeros[:] = full_row.unsqueeze(0)self.qzeros = qzeros.cpu()- def pack(self, linear, scales, zeros, g_idx=None):+ def pack(self, linear, scales, zeros, g_idx=None, device=None):if self.bits in [2, 4, 8]:- return self.pack_248_bits(linear, scales, zeros, g_idx)+ return self.pack_248_bits(linear, scales, zeros, g_idx, device)elif self.bits in [3]:- return self.pack_3bits(linear, scales, zeros, g_idx)+ return self.pack_3bits(linear, scales, zeros, g_idx, device)else:raise ValueError("Only 2,3,4,8 bits are supported.")diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex 1ddbb363e..244159ef0 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -21,6 +21,8 @@ import torchimport torch.nn as nnimport transformers+from auto_round.utils import _get_packing_device+logger = getLogger(__name__)@@ -88,16 +90,12 @@ class QuantLinear(nn.Module):pass# @torch.compile()- def pack_248_bits(self, linear, scales, zeros, g_idx=None):+ def pack_248_bits(self, linear, scales, zeros, g_idx=None, device=None):+ device = _get_packing_device(device)scales_t = scales.t().contiguous()if linear.bias is not None:self.bias = linear.bias.clone().half()self.scales = scales_t.clone().half()- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):@@ -150,16 +148,12 @@ class QuantLinear(nn.Module):qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * valueself.qzeros = qzeros.cpu()- def pack_3bits(self, linear, scales, zeros, g_idx=None):+ def pack_3bits(self, linear, scales, zeros, g_idx=None, device=None):+ device = _get_packing_device(device)scales_t = scales.t().contiguous()if linear.bias is not None:self.bias = linear.bias.clone().half()self.scales = scales_t.clone().half()- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):@@ -268,11 +262,11 @@ class QuantLinear(nn.Module):qzeros[:] = full_row.unsqueeze(0)self.qzeros = qzeros.cpu()- def pack(self, linear, scales, zeros, g_idx=None):+ def pack(self, linear, scales, zeros, g_idx=None, device=None):if self.bits in [2, 4, 8]:- return self.pack_248_bits(linear, scales, zeros, g_idx)+ return self.pack_248_bits(linear, scales, zeros, g_idx, device)elif self.bits in [3]:- return self.pack_3bits(linear, scales, zeros, g_idx)+ return self.pack_3bits(linear, scales, zeros, g_idx, device)else:raise ValueError("Only 2,3,4,8 bits are supported.")diff --git a/auto_round_extension/triton/qlinear_tritonv2.py b/auto_round_extension/triton/qlinear_tritonv2.pyindex 05b6fea15..4cbf706b3 100644--- a/auto_round_extension/triton/qlinear_tritonv2.py+++ b/auto_round_extension/triton/qlinear_tritonv2.py@@ -20,6 +20,7 @@ import torchimport torch.nn as nnimport transformers+from auto_round.utils import _get_packing_devicefrom auto_round_extension.triton.triton_utils.mixin import TritonModuleMixinlogger = getLogger(__name__)@@ -102,16 +103,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):def post_init(self):pass- def pack(self, linear, scales, zeros, g_idx=None):+ def pack(self, linear, scales, zeros, g_idx=None, device=None):+ device = _get_packing_device(device)scales_t = scales.t().contiguous()if linear.bias is not None:self.bias = linear.bias.clone().half()self.scales = scales_t.clone().half()- device = "cpu"- if torch.cuda.is_available():- device = "cuda:0"- elif torch.xpu.is_available():- device = "xpu:0"W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):