PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
remove IPEX related code, doc, and test (#1787)
Xin Heb2584b2d8505
diff --git a/.claude/skills/add-inference-backend/SKILL.md b/.claude/skills/add-inference-backend/SKILL.mdindex 820b6d3d6..426de37ef 100644--- a/.claude/skills/add-inference-backend/SKILL.md+++ b/.claude/skills/add-inference-backend/SKILL.md@@ -1,6 +1,6 @@---name: add-inference-backend-description: "Add a new hardware inference backend to AutoRound for deploying quantized models (e.g., CUDA/Marlin, Triton, IPEX/CPU, HPU, ARK). Use when implementing QuantLinear kernels, registering backend capabilities, or enabling quantized model inference on a new hardware platform."+description: "Add a new hardware inference backend to AutoRound for deploying quantized models (e.g., CUDA/Marlin, Triton, CPU, HPU, ARK). Use when implementing QuantLinear kernels, registering backend capabilities, or enabling quantized model inference on a new hardware platform."---# Adding a New Inference Backend to AutoRound@@ -238,7 +238,6 @@ def test_your_backend_e2e(tiny_opt_model_path, dataloader):| `auto_round:torch_*` | CPU/CUDA | `torch/` | Pure PyTorch fallback || `auto_round:ark` | ARK | `ark/` | ARK accelerator kernels || HPU backends | HPU | `hpu/` | Habana Gaudi optimized |-| IPEX backends | CPU | `ipex/` | Intel Extension for PyTorch |## Key Registration Pointsdiff --git a/.claude/skills/readme.md b/.claude/skills/readme.mdindex 6353ec022..cdd4dc532 100644--- a/.claude/skills/readme.md+++ b/.claude/skills/readme.md@@ -22,7 +22,7 @@ include:- `add-vlm-model`: walks through adding support for a new Vision-Language Model,including template, calibration dataset, and block handler registration- `add-inference-backend`: guides integration of a new hardware inference backend- (e.g., CUDA, HPU, IPEX, Triton)+ (e.g., CUDA, HPU, Triton)- `review-pr`: provides a structured workflow for reviewing pull requests,including Chinese translation verificationdiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex b37bcfa33..049c35b3d 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -200,9 +200,9 @@ AWQ_FORMAT = ["auto_round:auto_awq"]LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]WOQ_DEFAULT_ACT_BITS = [None, 16, 32]-# CPU backends that target Intel/x86 (ark / ipex / auto_round_kernel) cannot+# CPU backends that target Intel/x86 (ark / auto_round_kernel) cannot# run on Apple Silicon. Restrict them so the MLX backend wins on Darwin and we-# don't try to load auto-round-lib / intel-extension-for-pytorch on macOS.+# don't try to load auto-round-lib on macOS._NON_DARWIN_SYSTEMS = ["linux", "windows"]# AutoGPTQ is no longer maintained, supports transformers < 5.0.0@@ -622,68 +622,6 @@ BackendInfos["auto_round_kernel_awq_xpu"] = BackendInfo(requirements=["torch>=2.8.0", "auto-round-lib"],)-BackendInfos["ipex_gptq_cpu"] = BackendInfo(- device=["cpu"],- sym=[True, False],- packing_format=GPTQ_FORMAT,- bits=[4],- group_size=None,- priority=5,- checkers=[],- compute_dtype=["float16", "bfloat16"],- data_type=["int"],- act_bits=WOQ_DEFAULT_ACT_BITS,- alias=["ipex"],- requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],- systems=_NON_DARWIN_SYSTEMS, # intel-extension-for-pytorch is Intel x86 only-)--BackendInfos["ipex_gptq"] = BackendInfo(- device=["xpu"],- sym=[True, False],- packing_format=GPTQ_FORMAT,- bits=[4],- group_size=None,- priority=5,- checkers=[],- compute_dtype=["float16", "bfloat16"],- data_type=["int"],- act_bits=WOQ_DEFAULT_ACT_BITS,- alias=["ipex"],- requirements=["intel-extension-for-pytorch>=2.5"],-)--BackendInfos["ipex_awq_cpu"] = BackendInfo(- device=["cpu"],- sym=[True, False],- packing_format=AWQ_FORMAT,- bits=[4],- group_size=None,- priority=5,- checkers=[],- compute_dtype=["float16", "bfloat16"],- data_type=["int"],- act_bits=WOQ_DEFAULT_ACT_BITS,- alias=["ipex"],- requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],- systems=_NON_DARWIN_SYSTEMS,-)--BackendInfos["ipex_awq"] = BackendInfo(- device=["xpu"],- sym=[True, False],- packing_format=AWQ_FORMAT,- bits=[4],- group_size=None,- priority=5,- checkers=[],- compute_dtype=["float16", "bfloat16"],- data_type=["int"],- act_bits=WOQ_DEFAULT_ACT_BITS,- alias=["ipex"],- requirements=["intel-extension-for-pytorch>=2.5"],-)-BackendInfos["hpu"] = BackendInfo(device=["hpu"],sym=[True, False],@@ -858,16 +796,6 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):else: # auto_round must be at the endreturn qlinear.QuantLinear- if "ipex_gptq" in backend:- from auto_round_extension.ipex.qlinear_ipex_gptq import QuantLinear-- return QuantLinear-- if "ipex_awq" in backend:- from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear-- return QuantLinear-if "hpu" in backend:try:import habana_frameworks.torch.hpu # pylint: disable=E0401@@ -1193,7 +1121,7 @@ def get_highest_priority_backend(supported_backends = []for key in BackendInfos.keys():backend = BackendInfos[key]- # Filter by operating system (e.g. MLX is Darwin-only; ark/ipex CPU+ # Filter by operating system (e.g. MLX is Darwin-only; ark CPU# backends are non-Darwin only).if backend.systems is not None:if current_system not in [s.lower() for s in backend.systems]:@@ -1277,13 +1205,6 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="# Instructional messagesinstall_instructions = []- for cmd in pip_cmds:- if "intel-extension-for-pytorch" in cmd and target_device == "xpu":- install_instructions.append(- "Please refer to https://pytorch-extension.intel.com/installation?platform=gpu "- "to install intel-extension-for-pytorch. Ensure that the version matches your installed PyTorch."- )-prefix_msg = ("Better backend is found, please install all the following requirements to enable it."if logger_level != "error"diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex d3e702d11..112d53629 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -588,7 +588,7 @@ def convert_gptq_v1_to_v2_format(model: nn.Module):def _maybe_convert_gptq_to_mlx(model: nn.Module, used_backends: list[str]) -> None:"""On macOS with MLX available, convert GPTQ-format QuantLinear layers to MLX QuantLinearMLX.- This is the MLX equivalent of the IPEX/ARK post_init step: when an MLX backend was+ This is the MLX equivalent of the ARK post_init step: when an MLX backend wasselected but the checkpoint layers were materialized in GPTQ packing format, were-pack them into the MLX format so that ``mx.quantized_matmul`` can be used forhardware-accelerated inference on Apple Silicon. All conversion logic lives in@@ -654,7 +654,7 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:"""Performs post-initialization for different quantization backends.This function handles backend-specific post-init steps, including AutoGPTQ,- GPTQModel, IPEX layers, and ExLLaMAv2 kernels. It also ensures the+ GPTQModel, and ExLLaMAv2 kernels. It also ensures themodel's data type is compatible with all used backends.Args:@@ -668,7 +668,7 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:need_autogptq_init = Falseneed_gptqmodel_init = False- need_ipex_init = False+ need_ark_init = Falseused_gptq_exllamav2 = False# Determine which backends require post-initfor backend in used_backends:@@ -678,13 +678,8 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:used_gptq_exllamav2 = Trueelif backend.startswith("gptqmodel"):need_gptqmodel_init = True- elif backend.startswith(("ipex", "auto_round_kernel")):- need_ipex_init = True- if backend.startswith("ipex"):- logger.warning_once(- f"Backend '{backend}' is deprecated and will be removed in a future release. "- "Please use the 'ark' backend instead (requires auto-round-lib and torch>=2.8.0)."- )+ elif backend.startswith("auto_round_kernel"):+ need_ark_init = True# AutoGPTQ post-initif need_autogptq_init:@@ -716,12 +711,12 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:m.validate_once()model = gptq_post_init(model, use_act_order=False)- # IPEX post-init- if need_ipex_init:+ # ARK post-init+ if need_ark_init:message = "repacking to CPU/XPU format"- layers = [] ## ipex post_init will add one more layer+ layers = [] ## ark post_init will add one more layerfor n, m in model.named_modules():- if hasattr(m, "QUANT_TYPE") and ("ark" in m.QUANT_TYPE or "ipex" in m.QUANT_TYPE):+ if hasattr(m, "QUANT_TYPE") and "ark" in m.QUANT_TYPE:layers.append(m)for layer in tqdm(layers, desc=message, total=len(layers), leave=True):diff --git a/auto_round_extension/ipex/__init__.py b/auto_round_extension/ipex/__init__.pydeleted file mode 100644index 2929160d8..000000000--- a/auto_round_extension/ipex/__init__.py+++ /dev/null@@ -1,20 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear as IpexAWQQuantLinear-from auto_round_extension.ipex.qlinear_ipex_gptq import (- QuantLinear as IpexGPTQQuantLinear,-)--ipex_qlinear_classes = (IpexAWQQuantLinear, IpexGPTQQuantLinear)diff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.pydeleted file mode 100644index b521881ff..000000000--- a/auto_round_extension/ipex/qlinear_ipex_awq.py+++ /dev/null@@ -1,113 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import torch-import torch.nn as nn---class QuantLinear(nn.Module):- QUANT_TYPE = "ipex_awq"-- def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):- super().__init__()- assert w_bit == 4, "Only 4 bit are supported for now."- self.compute_dtype = torch.float16 if torch.xpu.is_available() else torch.bfloat16- self.in_features = in_features- self.out_features = out_features- self.w_bit = w_bit- self.group_size = group_size if group_size != -1 else in_features- self.scale_dtype = torch.float32-- # quick sanity check (make sure alignment)- assert self.in_features % self.group_size == 0- assert out_features % (32 // self.w_bit) == 0- self.pack_num = 32 // self.w_bit-- self.register_buffer(- "qzeros",- torch.zeros(- (in_features // self.group_size, out_features // self.pack_num),- dtype=torch.int32,- device=dev,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (in_features // self.group_size, out_features),- dtype=self.compute_dtype,- device=dev,- ),- )- if bias:- self.register_buffer(- "bias",- torch.zeros((out_features), dtype=self.compute_dtype, device=dev),- )- else:- self.register_buffer(- "bias",- None,- )- qweight = torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32, device=dev)- self.register_buffer("qweight", qweight)-- def post_init(self):- assert self.qweight.device.type == "cpu" or self.qweight.device.type == "xpu"- import intel_extension_for_pytorch as ipex-- self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(- self.qweight,- self.scales,- self.qzeros,- self.in_features,- self.out_features,- None,- self.bias,- self.group_size,- None,- 1,- 0,- )-- @classmethod- def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None):- awq_linear = cls(- w_bit,- group_size,- linear.in_features,- linear.out_features,- linear.bias is not None,- linear.weight.device,- )- if init_only: # just prepare for loading sd- return awq_linear-- raise NotImplementedError("Only inference is supported for IPEX kernels")-- @torch.no_grad()- def forward(self, x):-- outputs = self.ipex_linear(x)-- return outputs-- def extra_repr(self) -> str:- return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(- self.in_features,- self.out_features,- self.bias is not None,- self.w_bit,- self.group_size,- )diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.pydeleted file mode 100644index 7c5178a80..000000000--- a/auto_round_extension/ipex/qlinear_ipex_gptq.py+++ /dev/null@@ -1,390 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import math-from logging import getLogger--import numpy as np-import torch-import torch.nn as nn-import transformers--logger = getLogger(__name__)--BITS_DTYPE_MAPPING = {- 8: "int8",-}---def convert_dtype_torch2str(dtype):- if dtype == torch.int8:- return "int8"- elif dtype == torch.float:- return "fp32"- elif dtype == torch.float16:- return "fp16"- elif dtype == torch.bfloat16:- return "bf16"- elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:- return dtype- else:- assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)---class QuantLinear(nn.Module):- QUANT_TYPE = "ipex_gptq"-- def __init__(- self,- bits,- group_size,- infeatures,- outfeatures,- bias,- kernel_switch_threshold=128,- training=False,- weight_dtype=torch.bfloat16,- **kwargs,- ):- super().__init__()-- if bits not in [4]:- raise NotImplementedError("Only 4-bits is supported for IPEX.")-- self.infeatures = infeatures- self.outfeatures = outfeatures- self.bits = bits- self.group_size = group_size if group_size != -1 else infeatures- self.maxq = 2**self.bits - 1- self.weight_dtype = weight_dtype- self.asym = True-- self.register_buffer(- "qweight",- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),- )- self.register_buffer(- "qzeros",- torch.zeros(- (- math.ceil(infeatures / self.group_size),- outfeatures // 32 * self.bits,- ),- dtype=torch.int32,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (math.ceil(infeatures / self.group_size), outfeatures),- dtype=weight_dtype,- ),- )- # self.register_buffer(- # "g_idx",- # torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),- # )- if bias:- self.register_buffer("bias", torch.zeros((outfeatures), dtype=weight_dtype))- else:- self.bias = None-- self.kernel_switch_threshold = kernel_switch_threshold-- self.training = training-- # for training forward- self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)-- def post_init(self):- assert self.qweight.device.type == "cpu" or self.qweight.device.type == "xpu"-- # if not self.training and IPEX_AVAILABLE:- if not self.training:- import intel_extension_for_pytorch as ipex- from intel_extension_for_pytorch.nn.modules.weight_only_quantization import (- QuantDtype,- QuantMethod,- WeightOnlyQuantizedLinear,- )- from packaging import version-- from auto_round.utils import get_library_version-- ipex_version = get_library_version("intel_extension_for_pytorch")- if version.parse(ipex_version) >= version.parse("2.5"):- self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(- self.qweight,- self.scales,- self.qzeros,- self.infeatures,- self.outfeatures,- None,- self.bias,- self.group_size,- None,- ipex.llm.quantization.QuantMethod.GPTQ_GEMM,- ipex.llm.quantization.QuantDtype.INT4,- )- else:- import intel_extension_for_pytorch as ipex- from intel_extension_for_pytorch.nn.modules import WeightOnlyQuantizedLinear as ipex_linear- from intel_extension_for_pytorch.utils.weight_only_quantization import (- _convert_optimum_format_to_desired,- )-- qweight = self.qweight- scales = self.scales- qzeros = self.qzeros-- qweight, scales, qzeros = _convert_optimum_format_to_desired(qweight, scales, qzeros)- weight_dtype = {- 4: ipex.quantization.WoqWeightDtype.INT4,- 8: ipex.quantization.WoqWeightDtype.INT8,- }- compute_dtype = {- "fp32": ipex.quantization.WoqLowpMode.NONE, # follow the activation datatype.- "bf16": ipex.quantization.WoqLowpMode.BF16,- "fp16": ipex.quantization.WoqLowpMode.FP16,- "int8": ipex.quantization.WoqLowpMode.INT8,- }- ipex_qconfig_mapping = ipex.quantization.get_weight_only_quant_qconfig_mapping(- weight_dtype=weight_dtype[4],- lowp_mode=compute_dtype["bf16"],- act_quant_mode=ipex.quantization.WoqActQuantMode.PER_IC_BLOCK,- group_size=self.group_size,- )- tmp_linear = torch.nn.Linear(- self.infeatures,- self.outfeatures,- True if hasattr(self, "bias") and self.bias is not None else False,- )- if tmp_linear.bias is not None and self.bias is not None:- tmp_linear.bias = torch.nn.Parameter(self.bias.float())-- tmp_linear.qconfig = ipex_qconfig_mapping.global_qconfig- self.ipex_linear = ipex_linear.from_float_and_int4_weight(- mod=tmp_linear,- qweight=qweight,- scales=scales,- zero_points=qzeros,- bias=(self.bias.float() if hasattr(self, "bias") and self.bias is not None else None),- group_size=self.group_size,- g_idx=(self.g_idx if hasattr(self, "g_idx") else None),- )-- def pack(self, linear, scales, zeros, g_idx=None):- W = linear.weight.data.clone()- if type(linear) == nn.Conv2d:- W = W.flatten(1)- if type(linear) == transformers.pytorch_utils.Conv1D:- W = W.t()-- # self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx-- scales = scales.t().contiguous()- zeros = zeros.t().contiguous()- scale_zeros = zeros * scales- self.scales = scales.clone().to(dtype=linear.weight.dtype)- if linear.bias is not None:- self.bias = linear.bias.clone().to(dtype=linear.weight.dtype)-- intweight = []- for idx in range(self.infeatures):- intweight.append(- torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[- :, None- ]- )- intweight = torch.cat(intweight, dim=1)- intweight = intweight.t().contiguous()- intweight = intweight.numpy().astype(np.uint32)-- i = 0- row = 0- qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)- while row < qweight.shape[0]:- if self.bits in [2, 4, 8]:- for j in range(i, i + (32 // self.bits)):- qweight[row] |= intweight[j] << (self.bits * (j - i))- i += 32 // self.bits- row += 1- elif self.bits == 3:- for j in range(i, i + 10):- qweight[row] |= intweight[j] << (3 * (j - i))- i += 10- qweight[row] |= intweight[i] << 30- row += 1- qweight[row] |= (intweight[i] >> 2) & 1- i += 1- for j in range(i, i + 10):- qweight[row] |= intweight[j] << (3 * (j - i) + 1)- i += 10- qweight[row] |= intweight[i] << 31- row += 1- qweight[row] |= (intweight[i] >> 1) & 0x3- i += 1- for j in range(i, i + 10):- qweight[row] |= intweight[j] << (3 * (j - i) + 2)- i += 10- row += 1- else:- raise NotImplementedError("Only 2,3,4,8 bits are supported.")-- qweight = qweight.astype(np.int32)- self.qweight = torch.from_numpy(qweight)-- zeros -= 1- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)- i = 0- col = 0- while col < qzeros.shape[1]:- if self.bits in [2, 4, 8]:- for j in range(i, i + (32 // self.bits)):- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))- i += 32 // self.bits- col += 1- elif self.bits == 3:- for j in range(i, i + 10):- qzeros[:, col] |= zeros[:, j] << (3 * (j - i))- i += 10- qzeros[:, col] |= zeros[:, i] << 30- col += 1- qzeros[:, col] |= (zeros[:, i] >> 2) & 1- i += 1- for j in range(i, i + 10):- qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 1)- i += 10- qzeros[:, col] |= zeros[:, i] << 31- col += 1- qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3- i += 1- for j in range(i, i + 10):- qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 2)- i += 10- col += 1- else:- raise NotImplementedError("Only 2,3,4,8 bits are supported.")-- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)-- def forward(self, x: torch.Tensor):- if not self.training and hasattr(self, "ipex_linear"):- return self.ipex_linear(x)-- out_shape = x.shape[:-1] + (self.outfeatures,)- x = x.reshape(-1, x.shape[-1])- x_dtype = x.dtype- zeros = torch.bitwise_right_shift(- torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),- self.wf.unsqueeze(0),- ).to(torch.int16)- zeros = torch.bitwise_and(zeros, (2**self.bits) - 1)-- zeros = zeros + 1- zeros = zeros.reshape(self.scales.shape)-- weight = torch.bitwise_right_shift(- torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),- self.wf.unsqueeze(-1),- ).to(torch.int16)- weight = torch.bitwise_and(weight, (2**self.bits) - 1)-- weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])- num_itr = self.g_idx.shape[0] // x.shape[-1]- if num_itr == 1:- weights = self.scales[self.g_idx.long()] * (weight - zeros[self.g_idx.long()])- else:- num_dim = self.g_idx.shape[0] // num_itr- weights = []- for i in range(num_itr):- scale_i = self.scales[:, i * num_dim : (i + 1) * num_dim]- weight_i = weight[:, i * num_dim : (i + 1) * num_dim]- zeros_i = zeros[:, i * num_dim : (i + 1) * num_dim]- g_idx_i = self.g_idx[i * num_dim : (i + 1) * num_dim]- weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))- weights = torch.cat(weights, dim=1)- out = torch.matmul(x, weights)- out = out.to(x_dtype)- out = out.reshape(out_shape)- out = out + self.bias if self.bias is not None else out-- return out--[email protected]_grad()-def unpack_to_8bit_signed(qweight, qzeros, bits, g_idx=None):- wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32).unsqueeze(0)- zeros = None- if not torch.all(torch.eq(qzeros, 2004318071 if bits == 4 else 0b01111111011111110111111101111111)):- zp_shape = list(qzeros.shape)- zp_shape[1] = zp_shape[1] * (32 // bits)-- zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(- torch.int16 if bits == 8 else torch.int8- )- torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)- if bits == 8:- zeros = zeros.to(torch.uint8)- zeros = zeros + 1- try:- zeros = zeros.reshape(zp_shape)- except:- # zeros and scales have different iteam numbers.- # remove 1 (due to 0 + 1 in line 252)- zeros = zeros[zeros != 1]- zeros = zeros.reshape(zp_shape)-- weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(- torch.int16 if bits == 8 else torch.int8- )- weight.bitwise_and_((2**bits) - 1)- weight = weight.view(-1, weight.shape[-1])-- if g_idx is not None:- group_size = weight.shape[0] // qzeros.shape[0]- weight2 = weight.clone()- group_dict = {}- for i in range(len(g_idx)):- group_idx = g_idx[i].item()- if group_idx not in group_dict:- target_idx = group_idx * group_size- group_dict[group_idx] = 0- else:- group_dict[group_idx] = group_dict[group_idx] + 1- target_idx = group_idx * group_size + group_dict[group_idx]- weight2[target_idx] = weight[i]- weight = weight2-- return weight, zeros---# Copied from qlinear_marlin.py[email protected]_grad()-def dequantize_weight(qweight, qzeros, scales, bits):- unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)- group_size = unpacked_qweight.shape[0] // scales.shape[0]- scales = scales.repeat_interleave(group_size, dim=0)- if unpacked_qzeros is not None:- unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)- else:- unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32)- unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales-- return unpacked_qweight, unpacked_qzeros---__all__ = ["QuantLinear"]diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex dfb55e7a8..9762659f6 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -783,7 +783,6 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 || ark | xpu | 4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib<br/>torch>=2.8.0 || ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 |-| ipex(To be deprecated) | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch || marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel || exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq/gptq_zp+-1 | gptqmodel || exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 3 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex e04021a03..ee0abe1ee 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -744,7 +744,6 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 || ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib<br/>torch>=2.8.0 || ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 |-| ipex(即将废弃) | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch || marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel || exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel || exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |diff --git a/test/envs.py b/test/envs.pyindex d071b6668..5b6b5730c 100644--- a/test/envs.py+++ b/test/envs.py@@ -43,14 +43,6 @@ def is_compressed_tensors_available():return importlib.util.find_spec("compressed_tensors") is not None-def is_ipex_available():- try:- require_version("intel-extension-for-pytorch>=2.5")- return True- except ImportError:- return False--def is_flash_attn_available():return importlib.util.find_spec("flash_attn") is not None@@ -119,16 +111,6 @@ def require_awq(test_case):return unittest.skipUnless(is_awq_available(), "test requires autoawq")(test_case)-def require_ipex(test_case):- """- Decorator marking a test that requires intel-extension-for-pytorch.-- These tests are skipped when intel-extension-for-pytorch isn't installed.-- """- return unittest.skipUnless(is_ipex_available(), "test requires intel-extension-for-pytorch>=2.5")(test_case)--def require_optimum(test_case):"""Decorator marking a test that optimum.diff --git a/test/test_cuda/export/test_auto_round_format.py b/test/test_cuda/export/test_auto_round_format.pyindex 863dfc9e9..bb4f19e94 100644--- a/test/test_cuda/export/test_auto_round_format.py+++ b/test/test_cuda/export/test_auto_round_format.py@@ -13,7 +13,6 @@ from ...envs import (require_awq,require_gptqmodel,require_greater_than_050,- require_ipex,)from ...helpers import eval_generated_prompt, evaluate_accuracy, get_model_path, is_cuda_support_fp8@@ -131,93 +130,6 @@ class TestAutoRound:eval_generated_prompt(model, tokenizer)torch.cuda.empty_cache()- @pytest.mark.skip_ci(reason="IPEX is deprecated.")- @require_ipex- def test_autoround_gptq_sym_format(self, tiny_opt_model_path, dataloader):- bits, group_size, sym = 4, 128, True- autoround = AutoRound(- tiny_opt_model_path,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=dataloader,- )- quantized_model_path = self.save_dir-- _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)-- from transformers import AutoRoundConfig-- quantization_config = AutoRoundConfig(backend="ipex")-- model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config- )- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert "!!!" not in res-- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert "!!!" not in res-- model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config- )- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert "!!!" not in res-- @pytest.mark.skip_ci(reason="IPEX is deprecated.")- @require_awq- @require_ipex- def test_autoround_awq_sym_format(self, tiny_opt_model_path, dataloader):- bits, group_size, sym = 4, 128, True- autoround = AutoRound(- tiny_opt_model_path,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=dataloader,- )- quantized_model_path = self.save_dir-- _, quantized_model_path = autoround.quantize_and_save(- output_dir=quantized_model_path, format="auto_round:auto_awq"- )-- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert "!!!" not in res-- model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, device_map="cpu", trust_remote_code=True, torch_dtype=torch.bfloat16- )- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert "!!!" not in res-def test_fp8_block_fp8_format(self):model_name = "Qwen/Qwen3-0.6B"