Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

remove IPEX related code, doc, and test (#1787)

Xin Heb2584b2d8505
diff --git a/.claude/skills/add-inference-backend/SKILL.md b/.claude/skills/add-inference-backend/SKILL.md
index 820b6d3d6..426de37ef 100644
--- a/.claude/skills/add-inference-backend/SKILL.md
+++ b/.claude/skills/add-inference-backend/SKILL.md
@@ -1,6 +1,6 @@
---
name: add-inference-backend
-description: "Add a new hardware inference backend to AutoRound for deploying quantized models (e.g., CUDA/Marlin, Triton, IPEX/CPU, HPU, ARK). Use when implementing QuantLinear kernels, registering backend capabilities, or enabling quantized model inference on a new hardware platform."
+description: "Add a new hardware inference backend to AutoRound for deploying quantized models (e.g., CUDA/Marlin, Triton, CPU, HPU, ARK). Use when implementing QuantLinear kernels, registering backend capabilities, or enabling quantized model inference on a new hardware platform."
---
# Adding a New Inference Backend to AutoRound
@@ -238,7 +238,6 @@ def test_your_backend_e2e(tiny_opt_model_path, dataloader):
| `auto_round:torch_*` | CPU/CUDA | `torch/` | Pure PyTorch fallback |
| `auto_round:ark` | ARK | `ark/` | ARK accelerator kernels |
| HPU backends | HPU | `hpu/` | Habana Gaudi optimized |
-| IPEX backends | CPU | `ipex/` | Intel Extension for PyTorch |
## Key Registration Points
diff --git a/.claude/skills/readme.md b/.claude/skills/readme.md
index 6353ec022..cdd4dc532 100644
--- a/.claude/skills/readme.md
+++ b/.claude/skills/readme.md
@@ -22,7 +22,7 @@ include:
- `add-vlm-model`: walks through adding support for a new Vision-Language Model,
including template, calibration dataset, and block handler registration
- `add-inference-backend`: guides integration of a new hardware inference backend
- (e.g., CUDA, HPU, IPEX, Triton)
+ (e.g., CUDA, HPU, Triton)
- `review-pr`: provides a structured workflow for reviewing pull requests,
including Chinese translation verification
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index b37bcfa33..049c35b3d 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -200,9 +200,9 @@ AWQ_FORMAT = ["auto_round:auto_awq"]
LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
WOQ_DEFAULT_ACT_BITS = [None, 16, 32]
-# CPU backends that target Intel/x86 (ark / ipex / auto_round_kernel) cannot
+# CPU backends that target Intel/x86 (ark / auto_round_kernel) cannot
# run on Apple Silicon. Restrict them so the MLX backend wins on Darwin and we
-# don't try to load auto-round-lib / intel-extension-for-pytorch on macOS.
+# don't try to load auto-round-lib on macOS.
_NON_DARWIN_SYSTEMS = ["linux", "windows"]
# AutoGPTQ is no longer maintained, supports transformers < 5.0.0
@@ -622,68 +622,6 @@ BackendInfos["auto_round_kernel_awq_xpu"] = BackendInfo(
requirements=["torch>=2.8.0", "auto-round-lib"],
)
-BackendInfos["ipex_gptq_cpu"] = BackendInfo(
- device=["cpu"],
- sym=[True, False],
- packing_format=GPTQ_FORMAT,
- bits=[4],
- group_size=None,
- priority=5,
- checkers=[],
- compute_dtype=["float16", "bfloat16"],
- data_type=["int"],
- act_bits=WOQ_DEFAULT_ACT_BITS,
- alias=["ipex"],
- requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],
- systems=_NON_DARWIN_SYSTEMS, # intel-extension-for-pytorch is Intel x86 only
-)
-
-BackendInfos["ipex_gptq"] = BackendInfo(
- device=["xpu"],
- sym=[True, False],
- packing_format=GPTQ_FORMAT,
- bits=[4],
- group_size=None,
- priority=5,
- checkers=[],
- compute_dtype=["float16", "bfloat16"],
- data_type=["int"],
- act_bits=WOQ_DEFAULT_ACT_BITS,
- alias=["ipex"],
- requirements=["intel-extension-for-pytorch>=2.5"],
-)
-
-BackendInfos["ipex_awq_cpu"] = BackendInfo(
- device=["cpu"],
- sym=[True, False],
- packing_format=AWQ_FORMAT,
- bits=[4],
- group_size=None,
- priority=5,
- checkers=[],
- compute_dtype=["float16", "bfloat16"],
- data_type=["int"],
- act_bits=WOQ_DEFAULT_ACT_BITS,
- alias=["ipex"],
- requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],
- systems=_NON_DARWIN_SYSTEMS,
-)
-
-BackendInfos["ipex_awq"] = BackendInfo(
- device=["xpu"],
- sym=[True, False],
- packing_format=AWQ_FORMAT,
- bits=[4],
- group_size=None,
- priority=5,
- checkers=[],
- compute_dtype=["float16", "bfloat16"],
- data_type=["int"],
- act_bits=WOQ_DEFAULT_ACT_BITS,
- alias=["ipex"],
- requirements=["intel-extension-for-pytorch>=2.5"],
-)
-
BackendInfos["hpu"] = BackendInfo(
device=["hpu"],
sym=[True, False],
@@ -858,16 +796,6 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):
else: # auto_round must be at the end
return qlinear.QuantLinear
- if "ipex_gptq" in backend:
- from auto_round_extension.ipex.qlinear_ipex_gptq import QuantLinear
-
- return QuantLinear
-
- if "ipex_awq" in backend:
- from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear
-
- return QuantLinear
-
if "hpu" in backend:
try:
import habana_frameworks.torch.hpu # pylint: disable=E0401
@@ -1193,7 +1121,7 @@ def get_highest_priority_backend(
supported_backends = []
for key in BackendInfos.keys():
backend = BackendInfos[key]
- # Filter by operating system (e.g. MLX is Darwin-only; ark/ipex CPU
+ # Filter by operating system (e.g. MLX is Darwin-only; ark CPU
# backends are non-Darwin only).
if backend.systems is not None:
if current_system not in [s.lower() for s in backend.systems]:
@@ -1277,13 +1205,6 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="
# Instructional messages
install_instructions = []
- for cmd in pip_cmds:
- if "intel-extension-for-pytorch" in cmd and target_device == "xpu":
- install_instructions.append(
- "Please refer to https://pytorch-extension.intel.com/installation?platform=gpu "
- "to install intel-extension-for-pytorch. Ensure that the version matches your installed PyTorch."
- )
-
prefix_msg = (
"Better backend is found, please install all the following requirements to enable it."
if logger_level != "error"
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index d3e702d11..112d53629 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -588,7 +588,7 @@ def convert_gptq_v1_to_v2_format(model: nn.Module):
def _maybe_convert_gptq_to_mlx(model: nn.Module, used_backends: list[str]) -> None:
"""On macOS with MLX available, convert GPTQ-format QuantLinear layers to MLX QuantLinearMLX.
- This is the MLX equivalent of the IPEX/ARK post_init step: when an MLX backend was
+ This is the MLX equivalent of the ARK post_init step: when an MLX backend was
selected but the checkpoint layers were materialized in GPTQ packing format, we
re-pack them into the MLX format so that ``mx.quantized_matmul`` can be used for
hardware-accelerated inference on Apple Silicon. All conversion logic lives in
@@ -654,7 +654,7 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
"""Performs post-initialization for different quantization backends.
This function handles backend-specific post-init steps, including AutoGPTQ,
- GPTQModel, IPEX layers, and ExLLaMAv2 kernels. It also ensures the
+ GPTQModel, and ExLLaMAv2 kernels. It also ensures the
model's data type is compatible with all used backends.
Args:
@@ -668,7 +668,7 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
need_autogptq_init = False
need_gptqmodel_init = False
- need_ipex_init = False
+ need_ark_init = False
used_gptq_exllamav2 = False
# Determine which backends require post-init
for backend in used_backends:
@@ -678,13 +678,8 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
used_gptq_exllamav2 = True
elif backend.startswith("gptqmodel"):
need_gptqmodel_init = True
- elif backend.startswith(("ipex", "auto_round_kernel")):
- need_ipex_init = True
- if backend.startswith("ipex"):
- logger.warning_once(
- f"Backend '{backend}' is deprecated and will be removed in a future release. "
- "Please use the 'ark' backend instead (requires auto-round-lib and torch>=2.8.0)."
- )
+ elif backend.startswith("auto_round_kernel"):
+ need_ark_init = True
# AutoGPTQ post-init
if need_autogptq_init:
@@ -716,12 +711,12 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
m.validate_once()
model = gptq_post_init(model, use_act_order=False)
- # IPEX post-init
- if need_ipex_init:
+ # ARK post-init
+ if need_ark_init:
message = "repacking to CPU/XPU format"
- layers = [] ## ipex post_init will add one more layer
+ layers = [] ## ark post_init will add one more layer
for n, m in model.named_modules():
- if hasattr(m, "QUANT_TYPE") and ("ark" in m.QUANT_TYPE or "ipex" in m.QUANT_TYPE):
+ if hasattr(m, "QUANT_TYPE") and "ark" in m.QUANT_TYPE:
layers.append(m)
for layer in tqdm(layers, desc=message, total=len(layers), leave=True):
diff --git a/auto_round_extension/ipex/__init__.py b/auto_round_extension/ipex/__init__.py
deleted file mode 100644
index 2929160d8..000000000
--- a/auto_round_extension/ipex/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear as IpexAWQQuantLinear
-from auto_round_extension.ipex.qlinear_ipex_gptq import (
- QuantLinear as IpexGPTQQuantLinear,
-)
-
-ipex_qlinear_classes = (IpexAWQQuantLinear, IpexGPTQQuantLinear)
diff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.py
deleted file mode 100644
index b521881ff..000000000
--- a/auto_round_extension/ipex/qlinear_ipex_awq.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import torch
-import torch.nn as nn
-
-
-class QuantLinear(nn.Module):
- QUANT_TYPE = "ipex_awq"
-
- def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):
- super().__init__()
- assert w_bit == 4, "Only 4 bit are supported for now."
- self.compute_dtype = torch.float16 if torch.xpu.is_available() else torch.bfloat16
- self.in_features = in_features
- self.out_features = out_features
- self.w_bit = w_bit
- self.group_size = group_size if group_size != -1 else in_features
- self.scale_dtype = torch.float32
-
- # quick sanity check (make sure alignment)
- assert self.in_features % self.group_size == 0
- assert out_features % (32 // self.w_bit) == 0
- self.pack_num = 32 // self.w_bit
-
- self.register_buffer(
- "qzeros",
- torch.zeros(
- (in_features // self.group_size, out_features // self.pack_num),
- dtype=torch.int32,
- device=dev,
- ),
- )
- self.register_buffer(
- "scales",
- torch.zeros(
- (in_features // self.group_size, out_features),
- dtype=self.compute_dtype,
- device=dev,
- ),
- )
- if bias:
- self.register_buffer(
- "bias",
- torch.zeros((out_features), dtype=self.compute_dtype, device=dev),
- )
- else:
- self.register_buffer(
- "bias",
- None,
- )
- qweight = torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32, device=dev)
- self.register_buffer("qweight", qweight)
-
- def post_init(self):
- assert self.qweight.device.type == "cpu" or self.qweight.device.type == "xpu"
- import intel_extension_for_pytorch as ipex
-
- self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(
- self.qweight,
- self.scales,
- self.qzeros,
- self.in_features,
- self.out_features,
- None,
- self.bias,
- self.group_size,
- None,
- 1,
- 0,
- )
-
- @classmethod
- def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None):
- awq_linear = cls(
- w_bit,
- group_size,
- linear.in_features,
- linear.out_features,
- linear.bias is not None,
- linear.weight.device,
- )
- if init_only: # just prepare for loading sd
- return awq_linear
-
- raise NotImplementedError("Only inference is supported for IPEX kernels")
-
- @torch.no_grad()
- def forward(self, x):
-
- outputs = self.ipex_linear(x)
-
- return outputs
-
- def extra_repr(self) -> str:
- return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(
- self.in_features,
- self.out_features,
- self.bias is not None,
- self.w_bit,
- self.group_size,
- )
diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.py
deleted file mode 100644
index 7c5178a80..000000000
--- a/auto_round_extension/ipex/qlinear_ipex_gptq.py
+++ /dev/null
@@ -1,390 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import math
-from logging import getLogger
-
-import numpy as np
-import torch
-import torch.nn as nn
-import transformers
-
-logger = getLogger(__name__)
-
-BITS_DTYPE_MAPPING = {
- 8: "int8",
-}
-
-
-def convert_dtype_torch2str(dtype):
- if dtype == torch.int8:
- return "int8"
- elif dtype == torch.float:
- return "fp32"
- elif dtype == torch.float16:
- return "fp16"
- elif dtype == torch.bfloat16:
- return "bf16"
- elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:
- return dtype
- else:
- assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)
-
-
-class QuantLinear(nn.Module):
- QUANT_TYPE = "ipex_gptq"
-
- def __init__(
- self,
- bits,
- group_size,
- infeatures,
- outfeatures,
- bias,
- kernel_switch_threshold=128,
- training=False,
- weight_dtype=torch.bfloat16,
- **kwargs,
- ):
- super().__init__()
-
- if bits not in [4]:
- raise NotImplementedError("Only 4-bits is supported for IPEX.")
-
- self.infeatures = infeatures
- self.outfeatures = outfeatures
- self.bits = bits
- self.group_size = group_size if group_size != -1 else infeatures
- self.maxq = 2**self.bits - 1
- self.weight_dtype = weight_dtype
- self.asym = True
-
- self.register_buffer(
- "qweight",
- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
- )
- self.register_buffer(
- "qzeros",
- torch.zeros(
- (
- math.ceil(infeatures / self.group_size),
- outfeatures // 32 * self.bits,
- ),
- dtype=torch.int32,
- ),
- )
- self.register_buffer(
- "scales",
- torch.zeros(
- (math.ceil(infeatures / self.group_size), outfeatures),
- dtype=weight_dtype,
- ),
- )
- # self.register_buffer(
- # "g_idx",
- # torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
- # )
- if bias:
- self.register_buffer("bias", torch.zeros((outfeatures), dtype=weight_dtype))
- else:
- self.bias = None
-
- self.kernel_switch_threshold = kernel_switch_threshold
-
- self.training = training
-
- # for training forward
- self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)
-
- def post_init(self):
- assert self.qweight.device.type == "cpu" or self.qweight.device.type == "xpu"
-
- # if not self.training and IPEX_AVAILABLE:
- if not self.training:
- import intel_extension_for_pytorch as ipex
- from intel_extension_for_pytorch.nn.modules.weight_only_quantization import (
- QuantDtype,
- QuantMethod,
- WeightOnlyQuantizedLinear,
- )
- from packaging import version
-
- from auto_round.utils import get_library_version
-
- ipex_version = get_library_version("intel_extension_for_pytorch")
- if version.parse(ipex_version) >= version.parse("2.5"):
- self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(
- self.qweight,
- self.scales,
- self.qzeros,
- self.infeatures,
- self.outfeatures,
- None,
- self.bias,
- self.group_size,
- None,
- ipex.llm.quantization.QuantMethod.GPTQ_GEMM,
- ipex.llm.quantization.QuantDtype.INT4,
- )
- else:
- import intel_extension_for_pytorch as ipex
- from intel_extension_for_pytorch.nn.modules import WeightOnlyQuantizedLinear as ipex_linear
- from intel_extension_for_pytorch.utils.weight_only_quantization import (
- _convert_optimum_format_to_desired,
- )
-
- qweight = self.qweight
- scales = self.scales
- qzeros = self.qzeros
-
- qweight, scales, qzeros = _convert_optimum_format_to_desired(qweight, scales, qzeros)
- weight_dtype = {
- 4: ipex.quantization.WoqWeightDtype.INT4,
- 8: ipex.quantization.WoqWeightDtype.INT8,
- }
- compute_dtype = {
- "fp32": ipex.quantization.WoqLowpMode.NONE, # follow the activation datatype.
- "bf16": ipex.quantization.WoqLowpMode.BF16,
- "fp16": ipex.quantization.WoqLowpMode.FP16,
- "int8": ipex.quantization.WoqLowpMode.INT8,
- }
- ipex_qconfig_mapping = ipex.quantization.get_weight_only_quant_qconfig_mapping(
- weight_dtype=weight_dtype[4],
- lowp_mode=compute_dtype["bf16"],
- act_quant_mode=ipex.quantization.WoqActQuantMode.PER_IC_BLOCK,
- group_size=self.group_size,
- )
- tmp_linear = torch.nn.Linear(
- self.infeatures,
- self.outfeatures,
- True if hasattr(self, "bias") and self.bias is not None else False,
- )
- if tmp_linear.bias is not None and self.bias is not None:
- tmp_linear.bias = torch.nn.Parameter(self.bias.float())
-
- tmp_linear.qconfig = ipex_qconfig_mapping.global_qconfig
- self.ipex_linear = ipex_linear.from_float_and_int4_weight(
- mod=tmp_linear,
- qweight=qweight,
- scales=scales,
- zero_points=qzeros,
- bias=(self.bias.float() if hasattr(self, "bias") and self.bias is not None else None),
- group_size=self.group_size,
- g_idx=(self.g_idx if hasattr(self, "g_idx") else None),
- )
-
- def pack(self, linear, scales, zeros, g_idx=None):
- W = linear.weight.data.clone()
- if type(linear) == nn.Conv2d:
- W = W.flatten(1)
- if type(linear) == transformers.pytorch_utils.Conv1D:
- W = W.t()
-
- # self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
-
- scales = scales.t().contiguous()
- zeros = zeros.t().contiguous()
- scale_zeros = zeros * scales
- self.scales = scales.clone().to(dtype=linear.weight.dtype)
- if linear.bias is not None:
- self.bias = linear.bias.clone().to(dtype=linear.weight.dtype)
-
- intweight = []
- for idx in range(self.infeatures):
- intweight.append(
- torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[
- :, None
- ]
- )
- intweight = torch.cat(intweight, dim=1)
- intweight = intweight.t().contiguous()
- intweight = intweight.numpy().astype(np.uint32)
-
- i = 0
- row = 0
- qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)
- while row < qweight.shape[0]:
- if self.bits in [2, 4, 8]:
- for j in range(i, i + (32 // self.bits)):
- qweight[row] |= intweight[j] << (self.bits * (j - i))
- i += 32 // self.bits
- row += 1
- elif self.bits == 3:
- for j in range(i, i + 10):
- qweight[row] |= intweight[j] << (3 * (j - i))
- i += 10
- qweight[row] |= intweight[i] << 30
- row += 1
- qweight[row] |= (intweight[i] >> 2) & 1
- i += 1
- for j in range(i, i + 10):
- qweight[row] |= intweight[j] << (3 * (j - i) + 1)
- i += 10
- qweight[row] |= intweight[i] << 31
- row += 1
- qweight[row] |= (intweight[i] >> 1) & 0x3
- i += 1
- for j in range(i, i + 10):
- qweight[row] |= intweight[j] << (3 * (j - i) + 2)
- i += 10
- row += 1
- else:
- raise NotImplementedError("Only 2,3,4,8 bits are supported.")
-
- qweight = qweight.astype(np.int32)
- self.qweight = torch.from_numpy(qweight)
-
- zeros -= 1
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
- i = 0
- col = 0
- while col < qzeros.shape[1]:
- if self.bits in [2, 4, 8]:
- for j in range(i, i + (32 // self.bits)):
- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
- i += 32 // self.bits
- col += 1
- elif self.bits == 3:
- for j in range(i, i + 10):
- qzeros[:, col] |= zeros[:, j] << (3 * (j - i))
- i += 10
- qzeros[:, col] |= zeros[:, i] << 30
- col += 1
- qzeros[:, col] |= (zeros[:, i] >> 2) & 1
- i += 1
- for j in range(i, i + 10):
- qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 1)
- i += 10
- qzeros[:, col] |= zeros[:, i] << 31
- col += 1
- qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
- i += 1
- for j in range(i, i + 10):
- qzeros[:, col] |= zeros[:, j] << (3 * (j - i) + 2)
- i += 10
- col += 1
- else:
- raise NotImplementedError("Only 2,3,4,8 bits are supported.")
-
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
-
- def forward(self, x: torch.Tensor):
- if not self.training and hasattr(self, "ipex_linear"):
- return self.ipex_linear(x)
-
- out_shape = x.shape[:-1] + (self.outfeatures,)
- x = x.reshape(-1, x.shape[-1])
- x_dtype = x.dtype
- zeros = torch.bitwise_right_shift(
- torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),
- self.wf.unsqueeze(0),
- ).to(torch.int16)
- zeros = torch.bitwise_and(zeros, (2**self.bits) - 1)
-
- zeros = zeros + 1
- zeros = zeros.reshape(self.scales.shape)
-
- weight = torch.bitwise_right_shift(
- torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),
- self.wf.unsqueeze(-1),
- ).to(torch.int16)
- weight = torch.bitwise_and(weight, (2**self.bits) - 1)
-
- weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
- num_itr = self.g_idx.shape[0] // x.shape[-1]
- if num_itr == 1:
- weights = self.scales[self.g_idx.long()] * (weight - zeros[self.g_idx.long()])
- else:
- num_dim = self.g_idx.shape[0] // num_itr
- weights = []
- for i in range(num_itr):
- scale_i = self.scales[:, i * num_dim : (i + 1) * num_dim]
- weight_i = weight[:, i * num_dim : (i + 1) * num_dim]
- zeros_i = zeros[:, i * num_dim : (i + 1) * num_dim]
- g_idx_i = self.g_idx[i * num_dim : (i + 1) * num_dim]
- weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))
- weights = torch.cat(weights, dim=1)
- out = torch.matmul(x, weights)
- out = out.to(x_dtype)
- out = out.reshape(out_shape)
- out = out + self.bias if self.bias is not None else out
-
- return out
-
-
-def unpack_to_8bit_signed(qweight, qzeros, bits, g_idx=None):
- wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32).unsqueeze(0)
- zeros = None
- if not torch.all(torch.eq(qzeros, 2004318071 if bits == 4 else 0b01111111011111110111111101111111)):
- zp_shape = list(qzeros.shape)
- zp_shape[1] = zp_shape[1] * (32 // bits)
-
- zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(
- torch.int16 if bits == 8 else torch.int8
- )
- torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)
- if bits == 8:
- zeros = zeros.to(torch.uint8)
- zeros = zeros + 1
- try:
- zeros = zeros.reshape(zp_shape)
- except:
- # zeros and scales have different iteam numbers.
- # remove 1 (due to 0 + 1 in line 252)
- zeros = zeros[zeros != 1]
- zeros = zeros.reshape(zp_shape)
-
- weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(
- torch.int16 if bits == 8 else torch.int8
- )
- weight.bitwise_and_((2**bits) - 1)
- weight = weight.view(-1, weight.shape[-1])
-
- if g_idx is not None:
- group_size = weight.shape[0] // qzeros.shape[0]
- weight2 = weight.clone()
- group_dict = {}
- for i in range(len(g_idx)):
- group_idx = g_idx[i].item()
- if group_idx not in group_dict:
- target_idx = group_idx * group_size
- group_dict[group_idx] = 0
- else:
- group_dict[group_idx] = group_dict[group_idx] + 1
- target_idx = group_idx * group_size + group_dict[group_idx]
- weight2[target_idx] = weight[i]
- weight = weight2
-
- return weight, zeros
-
-
-# Copied from qlinear_marlin.py
-def dequantize_weight(qweight, qzeros, scales, bits):
- unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)
- group_size = unpacked_qweight.shape[0] // scales.shape[0]
- scales = scales.repeat_interleave(group_size, dim=0)
- if unpacked_qzeros is not None:
- unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)
- else:
- unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32)
- unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales
-
- return unpacked_qweight, unpacked_qzeros
-
-
-__all__ = ["QuantLinear"]
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index dfb55e7a8..9762659f6 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -783,7 +783,6 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 |
| ark | xpu | 4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib<br/>torch>=2.8.0 |
| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 |
-| ipex(To be deprecated) | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 3 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index e04021a03..ee0abe1ee 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -744,7 +744,6 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 |
| ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib<br/>torch>=2.8.0 |
| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 |
-| ipex(即将废弃) | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
| marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
diff --git a/test/envs.py b/test/envs.py
index d071b6668..5b6b5730c 100644
--- a/test/envs.py
+++ b/test/envs.py
@@ -43,14 +43,6 @@ def is_compressed_tensors_available():
return importlib.util.find_spec("compressed_tensors") is not None
-def is_ipex_available():
- try:
- require_version("intel-extension-for-pytorch>=2.5")
- return True
- except ImportError:
- return False
-
-
def is_flash_attn_available():
return importlib.util.find_spec("flash_attn") is not None
@@ -119,16 +111,6 @@ def require_awq(test_case):
return unittest.skipUnless(is_awq_available(), "test requires autoawq")(test_case)
-def require_ipex(test_case):
- """
- Decorator marking a test that requires intel-extension-for-pytorch.
-
- These tests are skipped when intel-extension-for-pytorch isn't installed.
-
- """
- return unittest.skipUnless(is_ipex_available(), "test requires intel-extension-for-pytorch>=2.5")(test_case)
-
-
def require_optimum(test_case):
"""
Decorator marking a test that optimum.
diff --git a/test/test_cuda/export/test_auto_round_format.py b/test/test_cuda/export/test_auto_round_format.py
index 863dfc9e9..bb4f19e94 100644
--- a/test/test_cuda/export/test_auto_round_format.py
+++ b/test/test_cuda/export/test_auto_round_format.py
@@ -13,7 +13,6 @@ from ...envs import (
require_awq,
require_gptqmodel,
require_greater_than_050,
- require_ipex,
)
from ...helpers import eval_generated_prompt, evaluate_accuracy, get_model_path, is_cuda_support_fp8
@@ -131,93 +130,6 @@ class TestAutoRound:
eval_generated_prompt(model, tokenizer)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="IPEX is deprecated.")
- @require_ipex
- def test_autoround_gptq_sym_format(self, tiny_opt_model_path, dataloader):
- bits, group_size, sym = 4, 128, True
- autoround = AutoRound(
- tiny_opt_model_path,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=dataloader,
- )
- quantized_model_path = self.save_dir
-
- _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)
-
- from transformers import AutoRoundConfig
-
- quantization_config = AutoRoundConfig(backend="ipex")
-
- model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config
- )
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert "!!!" not in res
-
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert "!!!" not in res
-
- model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config
- )
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert "!!!" not in res
-
- @pytest.mark.skip_ci(reason="IPEX is deprecated.")
- @require_awq
- @require_ipex
- def test_autoround_awq_sym_format(self, tiny_opt_model_path, dataloader):
- bits, group_size, sym = 4, 128, True
- autoround = AutoRound(
- tiny_opt_model_path,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=dataloader,
- )
- quantized_model_path = self.save_dir
-
- _, quantized_model_path = autoround.quantize_and_save(
- output_dir=quantized_model_path, format="auto_round:auto_awq"
- )
-
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert "!!!" not in res
-
- model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, device_map="cpu", trust_remote_code=True, torch_dtype=torch.bfloat16
- )
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert "!!!" not in res
-
def test_fp8_block_fp8_format(self):
model_name = "Qwen/Qwen3-0.6B"