PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support 5,6,7 bits on cuda and add humming backend (#2281)
Wenhua Cheng1f23576afd25
diff --git a/auto_round/compressors/mllm/dataset.py b/auto_round/compressors/mllm/dataset.pyindex 572227655..fec57f44a 100644--- a/auto_round/compressors/mllm/dataset.py+++ b/auto_round/compressors/mllm/dataset.py@@ -56,12 +56,21 @@ def register_dataset(name_list):class LlavaDataset(Dataset):"""Dataset for supervised fine-tuning."""+ LLAVA_REPO_ID = "liuhaotian/LLaVA-Instruct-150K"BASE_LLAVA_URL = "https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/"+ LLAVA_DATASET_FILES = {+ "llava_conv_58k": "conversation_58k.json",+ "llava_instruct_80k": "llava_instruct_80k.json",+ "llava_instruct_150k": "llava_instruct_150k.json",+ }+ # Kept for backward compatibility, `LLAVA_DATASET_FILES` is the source of truth.LLAVA_DATASET = {"llava_conv_58k": BASE_LLAVA_URL + "conversation_58k.json?download=true","llava_instruct_80k": BASE_LLAVA_URL + "llava_instruct_80k.json?download=true","llava_instruct_150k": BASE_LLAVA_URL + "llava_instruct_150k.json?download=true",}+ DOWNLOAD_TIMEOUT = 60+ DOWNLOAD_RETRIES = 3_COCO_DATA_URL = "http://images.cocodataset.org/train2017/"IMAGE_TOKEN = "<image>"MAX_SUPPORT_SEQLEN = 512@@ -88,18 +97,18 @@ class LlavaDataset(Dataset):logger.info(f"use dataset {dataset_path}, loading from disk...")self.questions = json.load(open(dataset_path, "r"))else:- import requests-if dataset_path == "liuhaotian/llava":dataset_path = "llava_conv_58k"else:dataset_path = dataset_path.split("/")[-1]dataset_name = dataset_path.split("/")[-1]- if dataset_name in self.LLAVA_DATASET:+ if dataset_name in self.LLAVA_DATASET_FILES:logger.info(f"use dataset {dataset_name}, downloading...")- self.questions = requests.get(self.LLAVA_DATASET[dataset_name], stream=True).json()+ self.questions = self._download_questions(dataset_name)else:- raise KeyError(f"{dataset_path} is not support, we support {self.LLAVA_DATASET.keys()}.")+ raise KeyError(+ f"{dataset_path} is not supported; supported datasets: {', '.join(self.LLAVA_DATASET_FILES.keys())}."+ )self.seqlen = seqlenself.questions = self.check(self.questions, self.seqlen, nsamples)@@ -117,6 +126,56 @@ class LlavaDataset(Dataset):image_fold = image_fold["image"]self.image_fold = image_fold+ @classmethod+ def _download_questions(cls, dataset_name: str):+ """Download a LLaVA instruction dataset and return the decoded JSON payload.++ ``huggingface_hub`` is preferred because it honours ``HF_TOKEN``/``HF_ENDPOINT`` mirrors,+ retries transient failures and caches the file locally. A plain HTTP request is used as a+ fallback. Both paths validate the payload, so a non-JSON body (an HTML error/rate-limit+ page, an empty response from a proxy, ...) raises an actionable error instead of a bare+ ``JSONDecodeError``.+ """+ filename = cls.LLAVA_DATASET_FILES[dataset_name]+ errors = []++ try:+ from huggingface_hub import hf_hub_download++ local_path = hf_hub_download(repo_id=cls.LLAVA_REPO_ID, filename=filename, repo_type="dataset")+ with open(local_path, encoding="utf-8") as f:+ return json.load(f)+ except Exception as e:+ errors.append(f"huggingface_hub: {type(e).__name__}: {e}")+ logger.warning(+ f"failed to fetch dataset '{dataset_name}' via huggingface_hub ({type(e).__name__}: {e}),"+ " falling back to a direct http download."+ )++ import requests++ url = cls.LLAVA_DATASET[dataset_name]+ for attempt in range(1, cls.DOWNLOAD_RETRIES + 1):+ try:+ response = requests.get(url, timeout=cls.DOWNLOAD_TIMEOUT)+ response.raise_for_status()+ return json.loads(response.content.decode("utf-8"))+ except Exception as e:+ errors.append(f"http attempt {attempt}: {type(e).__name__}: {e}")+ if attempt < cls.DOWNLOAD_RETRIES:+ logger.warning(+ f"failed to download dataset '{dataset_name}' from {url}"+ f" ({type(e).__name__}: {e}), retrying (attempt {attempt + 1}/{cls.DOWNLOAD_RETRIES})..."+ )++ details = "\n ".join(errors)+ raise RuntimeError(+ f"could not download the calibration dataset '{dataset_name}' from"+ f" '{cls.LLAVA_REPO_ID}'. Please check your network connection/proxy, or download"+ f" '{filename}' manually and pass its local path via the `dataset` argument."+ f"\nCauses:\n {details}"+ )+def check(self, questions, word_len, nsamples):def _check(questions, min_word_len, max_word_len, nsamples):new_questions = []diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex 86db8a3ee..662987529 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -40,6 +40,18 @@ import torch.nn as nnfrom torch.autograd import Functionfrom auto_round.utils import get_packing_device+from auto_round.utils.bit_packing import (+ pack_awq_bitstream,+ pack_scalar_zero,+ requires_generic_bit_packing,+ unpack_awq_bitstream,+)++# Bit-widths the AWQ packing layout can represent. AutoAWQ's classic layout+# interleaves 8 values per int32 word, which only lines up at 4 bits; 5/6/7-bit+# keep the same 8-value interleave but write it into the generic bit-stream+# layout (exactly what humming's AWQWeightSchema expects).+SUPPORTED_AWQ_BITS = (4, 5, 6, 7)def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):@@ -84,10 +96,16 @@ def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):def dequantize_gemm(qweight, qzeros, scales, bits, group_size):- # Unpack the qweight and qzeros tensors- iweight, izeros = unpack_awq(qweight, qzeros, bits)- # Reverse the order of the iweight and izeros tensors- iweight, izeros = reverse_awq_order(iweight, izeros, bits)+ if requires_generic_bit_packing(bits):+ # 5/6/7-bit: values straddle int32 words, so unpack via the generic+ # bit-stream helper (which also undoes the AWQ interleave).+ iweight = unpack_awq_bitstream(qweight, bits, dim=1)+ izeros = unpack_awq_bitstream(qzeros, bits, dim=1)+ else:+ # Unpack the qweight and qzeros tensors+ iweight, izeros = unpack_awq(qweight, qzeros, bits)+ # Reverse the order of the iweight and izeros tensors+ iweight, izeros = reverse_awq_order(iweight, izeros, bits)# overflow checksiweight = torch.bitwise_and(iweight, (2**bits) - 1)@@ -141,26 +159,32 @@ class WQLinear_GEMM(nn.Module):def __init__(self, w_bit, group_size, in_features, out_features, bias, dev, training=False):super().__init__()- if w_bit not in [4]:- raise NotImplementedError("Only 4-bit are supported for now.")+ if w_bit not in SUPPORTED_AWQ_BITS:+ raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_AWQ_BITS))}-bit are supported for now.")self.in_features = in_featuresself.out_features = out_featuresself.w_bit = w_bitself.group_size = group_size if group_size != -1 else in_featuresself.training = training+ self.use_generic_bit_packing = requires_generic_bit_packing(w_bit)# quick sanity check (make sure alignment)if self.in_features % self.group_size != 0:raise ValueError(f"in_features ({self.in_features}) shape mismatch")- if out_features % (32 // self.w_bit) != 0:- raise ValueError(f"out_features ({out_features}) shape mismatch")+ # 2/4/8-bit keep AutoAWQ's original ``32 // w_bit`` alignment rule; 5/6/7-bit+ # pack a contiguous bit-stream over blocks of 32 values, so the output+ # dimension must be a multiple of 32.+ pack_align = 32 if self.use_generic_bit_packing else 32 // self.w_bit+ if out_features % pack_align != 0:+ raise ValueError(f"out_features ({out_features}) must be a multiple of {pack_align} for {w_bit}-bit AWQ")+ packed_out_features = out_features * self.w_bit // 32self.register_buffer("qweight",torch.zeros(- (in_features, out_features // (32 // self.w_bit)),+ (in_features, packed_out_features),dtype=torch.int32,device=dev,),@@ -168,7 +192,7 @@ class WQLinear_GEMM(nn.Module):self.register_buffer("qzeros",torch.zeros(- (in_features // self.group_size, out_features // (32 // self.w_bit)),+ (in_features // self.group_size, packed_out_features),dtype=torch.int32,device=dev,),@@ -249,6 +273,22 @@ class WQLinear_GEMM(nn.Module):intweight = intweight.to(dtype=torch.int32)del repeat_scales+ if awq_linear.use_generic_bit_packing:+ # 5/6/7-bit: AWQ interleave (groups of 8) + generic bit-stream packing.+ awq_linear.qweight = pack_awq_bitstream(intweight, awq_linear.w_bit, dim=1).to("cpu")+ packed_cols = scales.shape[1] * awq_linear.w_bit // 32+ if isinstance(zeros, torch.Tensor):+ zeros = zeros.to(dtype=torch.int32, device=device)+ qzeros = pack_awq_bitstream(zeros, awq_linear.w_bit, dim=1)+ else:+ # A single shared zero point is interleave-invariant, so it can be+ # tiled directly without reordering.+ qzeros = pack_scalar_zero(+ int(zeros), awq_linear.w_bit, packed_cols, (scales.shape[0], packed_cols), device=device+ )+ awq_linear.qzeros = qzeros.to("cpu")+ return awq_linear+intweight = intweight.reshape(-1, intweight.shape[1] // pack_num, pack_num)new_order_map = torch.tensor([0, 4, 1, 5, 2, 6, 3, 7], device=device) * awq_linear.w_bitdiff --git a/auto_round/export/formats/backends/auto_awq.py b/auto_round/export/formats/backends/auto_awq.pyindex 1ab8c2831..00905cb5b 100644--- a/auto_round/export/formats/backends/auto_awq.py+++ b/auto_round/export/formats/backends/auto_awq.py@@ -25,13 +25,17 @@ from auto_round.schemes import QuantizationScheme@OutputFormat.register("auto_awq")class AutoAWQFormat(OutputFormat):- support_schemes = ["W4A16"]+ support_schemes = ["W4A16", "W5A16", "W6A16", "W7A16"]format_name = "auto_awq"+ # See AutoGPTQFormat.EXTENDED_BITS -- 5/6/7-bit uses the generic bit-stream+ # layout, which upstream AutoAWQ kernels cannot read.+ EXTENDED_BITS = (5, 6, 7)+@classmethoddef check_scheme_args(cls, scheme: QuantizationScheme) -> bool:error_logs = []- if scheme.bits != 4:+ if scheme.bits not in (4,) + cls.EXTENDED_BITS:error_logs.append(f"bits={scheme.bits}")if not re.search("int", scheme.data_type):error_logs.append(f"data_type={scheme.data_type}")@@ -49,14 +53,17 @@ class AutoAWQFormat(OutputFormat):@staticmethoddef check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):"""Check whether a model is compatible with the AutoAWQ GEMM kernel."""+ from auto_round.utils.bit_packing import requires_generic_bit_packingfrom auto_round.utils.model import get_layer_names_in_block, get_module- if bits != 4:+ if bits not in (4, 5, 6, 7):return False, "AutoAWQ GEMM kernel only supports 4 bits"for _, module in model.named_modules():if type(module) == transformers.pytorch_utils.Conv1D:return False, "AutoAWQ GEMM kernel does not support conv1d"+ # 5/6/7-bit packs a contiguous bit-stream over blocks of 32 values.+ out_align = 32 if requires_generic_bit_packing(bits) else 32 // bitslayer_names = get_layer_names_in_block(model)for layer_name in layer_names:if (@@ -69,8 +76,8 @@ class AutoAWQFormat(OutputFormat):layer = get_module(model, layer_name)if layer.in_features % group_size != 0:return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"- if layer.out_features % (32 // bits) != 0:- return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"+ if layer.out_features % out_align != 0:+ return False, f"Layer {layer_name} out_features is not multiple of {out_align}"return True, ""@@ -80,8 +87,17 @@ class AutoAWQFormat(OutputFormat):)if not awq_supported:logger.warning(f"The AutoAWQ format may not be supported due to {info}")- if scheme.bits != 4:- raise ValueError(f"auto_awq format support quantization scheme with W4A16 but got bits={scheme.bits}")+ if scheme.bits in self.EXTENDED_BITS and not self.output_format.startswith("auto_round"):+ raise ValueError(+ f"{self.output_format} format does not support bits={scheme.bits}. "+ f"{self.EXTENDED_BITS} bit packing is only defined for the `auto_round` format family, "+ "please export to `auto_round:auto_awq`."+ )+ if scheme.bits not in (4,) + self.EXTENDED_BITS:+ raise ValueError(+ f"{self.format_name} format support quantization scheme with {','.join(self.support_schemes)} "+ f"but got bits={scheme.bits}, please have a check."+ )if self.backend is None:ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)diff --git a/auto_round/export/formats/backends/auto_gptq.py b/auto_round/export/formats/backends/auto_gptq.pyindex 4fcaaca61..03a14a33d 100644--- a/auto_round/export/formats/backends/auto_gptq.py+++ b/auto_round/export/formats/backends/auto_gptq.py@@ -24,16 +24,36 @@ from auto_round.schemes import QuantizationScheme@OutputFormat.register("auto_gptq", "gptqmodel")class AutoGPTQFormat(OutputFormat):- support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32", "W4A16_MIXED"]+ support_schemes = [+ "W4A16",+ "W2A16",+ "W3A16",+ "W8A16",+ "BF16",+ "W2A16G64",+ "W2A16G32",+ "W4A16_MIXED",+ ]format_name = "auto_gptq"+ # 5/6/7-bit weights straddle int32 word boundaries. AutoRound packs them with the+ # generic bit-stream layout (see auto_round.utils.bit_packing), which upstream+ # AutoGPTQ/GPTQModel kernels cannot read -- so restrict them to ``auto_round:*``.+ EXTENDED_BITS = (5, 6, 7)+def check_and_reset_format(self, scheme: QuantizationScheme, ctx):+ if scheme.bits in self.EXTENDED_BITS and not self.output_format.startswith("auto_round"):+ raise ValueError(+ f"{self.output_format} format does not support bits={scheme.bits}. "+ f"{self.EXTENDED_BITS} bit packing is only defined for the `auto_round` format family, "+ "please export to `auto_round` or `auto_round:auto_gptq`."+ )if not scheme.sym:logger.warning("the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"" particularly for 2-bit quantization and smaller models."" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/3/4/8 bits)."+ "the AutoRound format(2/3/4/5/6/7/8 bits).")if self.backend is None:ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.pyindex 4d570b26f..84134f6df 100644--- a/auto_round/export/formats/backends/autoround.py+++ b/auto_round/export/formats/backends/autoround.py@@ -37,6 +37,9 @@ class AutoRoundFormat(OutputFormat):"W4A16_MIXED","W2A16","W3A16",+ "W5A16",+ "W6A16",+ "W7A16","W8A16","MXFP4","MXFP8",diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex dd6edd6fd..d45549a75 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -181,6 +181,31 @@ mxfp_nvfp_feature_checker = functools.partial(in_feature_checker_group_size)ark_feature_checker = functools.partial(in_feature_checker_group_size)+# humming repacks weights with ``padded_shape_n % 64 == 0`` / ``padded_shape_k % 32 == 0``.+# humming can pad internally; requiring 32-multiples matches the checkpoint packing constraints.+humming_feature_checker = functools.partial(+ feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32+)+++# humming is published only as a git repo (distribution name ``humming-kernels``),+# so it cannot be expressed as a PEP 440 requirement string like the other backends.+HUMMING_INSTALL_SPEC = "git+https://github.com/inclusionAI/humming.git"+++def humming_available_checker():+ """Requirement hook: humming needs both the package and a CUDA device.++ Returns ``(is_available, install_spec)``; the spec is fed to ``pip install`` in the+ "better backend found" hint, so it must be the git URL rather than a version spec.+ """+ try:+ from auto_round_extension.humming.qlinear_humming import is_humming_available++ return is_humming_available(), HUMMING_INSTALL_SPEC+ except Exception: # pragma: no cover - depends on the local environment+ return False, HUMMING_INSTALL_SPEC+def fp8_static_scheme_checker(in_feature: int,@@ -452,7 +477,7 @@ BackendInfos["auto_round:torch"] = BackendInfo(compute_dtype=["float16", "bfloat16"],data_type=["int"],act_bits=WOQ_DEFAULT_ACT_BITS,- bits=[2, 3, 4, 8],+ bits=[2, 3, 4, 5, 6, 7, 8],priority=0,checkers=[exllamav2_feature_checker],alias=["auto_round", "torch"],@@ -467,13 +492,62 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(compute_dtype=["float16", "bfloat16"],data_type=["int"],act_bits=WOQ_DEFAULT_ACT_BITS,- bits=[2, 3, 4, 8],+ bits=[2, 3, 4, 5, 6, 7, 8],priority=0,checkers=[exllamav2_feature_checker],alias=["torch", "torch_zp"],requirements=["auto-round>=0.5.1"],)+# humming: JIT-compiled CUDA GEMM kernels covering every weight bit-width from+# 2 to 8. This is the only fast path for 5/6/7-bit checkpoints, and it is also+# faster than the Triton kernels for the classic 2/3/4/8-bit ones. Priority is+# set above tritonv2 (2) but below marlin (6) so that existing 4-bit defaults+# keep using marlin where it is available.+BackendInfos["auto_round:humming"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format=GPTQ_FORMAT_NO_ZP,+ compute_dtype=["float16", "bfloat16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ bits=[2, 3, 4, 5, 6, 7, 8],+ priority=4,+ checkers=[humming_feature_checker],+ alias=["humming"],+ requirements=[humming_available_checker],+)++BackendInfos["auto_round:humming_zp"] = BackendInfo(+ device=["cuda"],+ # asym `auto_round:auto_gptq` stores ``zp - 1``, which humming does not model;+ # only the symmetric variant (where the zero point is implicit) is supported.+ sym=[True],+ packing_format=GPTQ_FORMAT,+ compute_dtype=["float16", "bfloat16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ bits=[2, 3, 4, 5, 6, 7, 8],+ priority=4,+ checkers=[humming_feature_checker],+ alias=["humming", "humming_zp"],+ requirements=[humming_available_checker],+)++BackendInfos["auto_round:humming_awq"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format=AWQ_FORMAT,+ compute_dtype=["float16", "bfloat16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ bits=[2, 3, 4, 5, 6, 7, 8],+ priority=4,+ checkers=[humming_feature_checker],+ alias=["humming", "humming_awq"],+ requirements=[humming_available_checker],+)+BackendInfos["gptqmodel:marlin"] = BackendInfo(device=["cuda"],sym=[True],@@ -853,6 +927,23 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):if "auto_round:fake" in backend:return ar_qmodules.FakeActQuantLinear+ if "humming" in backend:+ try:+ import humming # noqa: F401 # pylint: disable=E0401+ except Exception as e:+ raise ImportError(+ "Please install the humming kernels to use this backend, e.g.: "+ "`pip install git+https://github.com/inclusionAI/humming.git`"+ ) from e+ import auto_round_extension.humming.qlinear_humming as humming_qlinear++ if "awq" in backend:+ return humming_qlinear.QuantLinearAWQ+ elif "zp" in backend:+ return humming_qlinear.QuantLinearGPTQ+ else: # plain auto_round packing must be checked last+ return humming_qlinear.QuantLinear+if "auto_round_kernel" in backend or "ark" in backend:try:import auto_round_kernel as ark # pylint: disable=E0611, E0401@@ -1261,6 +1352,13 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="# Filter requirementsmissing_requirements = []for req in requirements:+ # Callable requirements (e.g. humming) report their own availability and+ # return the spec to hand to `pip install`.+ if callable(req):+ available, install_spec = req()+ if not available and install_spec:+ missing_requirements.append(install_spec)+ continuetry:require_version(req)except Exception:diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 491137040..7ebf1718b 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -523,7 +523,19 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,bias=bias,)+ if "humming" in layer_backend:+ return QuantLinear(+ bits=config["bits"],+ group_size=config["group_size"],+ infeatures=in_features,+ outfeatures=out_features,+ bias=bias,+ sym=config["sym"],+ weight_dtype=layer.weight.dtype,+ )+if "auto_round_kernel" in layer_backend:+return QuantLinear(bits=config["bits"],group_size=config["group_size"],@@ -717,6 +729,7 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:need_autogptq_init = Falseneed_gptqmodel_init = Falseneed_ark_init = False+ need_humming_init = Falseused_gptq_exllamav2 = False# Determine which backends require post-initfor backend in used_backends:@@ -728,6 +741,8 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:need_gptqmodel_init = Trueelif backend.startswith("auto_round_kernel"):need_ark_init = True+ elif "humming" in backend:+ need_humming_init = True# AutoGPTQ post-initif need_autogptq_init:@@ -770,6 +785,13 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:for layer in tqdm(layers, desc=message, total=len(layers), leave=True):layer.post_init()+ # humming post-init: repack the checkpoint layout into the kernel-native one+ # and JIT-compile the selected GEMM.+ if need_humming_init:+ layers = [m for _, m in model.named_modules() if getattr(m, "QUANT_TYPE", "").startswith("humming")]+ for layer in tqdm(layers, desc="repacking to humming format", total=len(layers), leave=True):+ layer.post_init()+# ExLLaMAv2 kernelsif used_gptq_exllamav2:_import_exllamav2_kernels()diff --git a/auto_round/utils/bit_packing.py b/auto_round/utils/bit_packing.pynew file mode 100644index 000000000..f444cc4d1--- /dev/null+++ b/auto_round/utils/bit_packing.py@@ -0,0 +1,248 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Generic little-endian bit-stream packing for arbitrary weight bit-widths (1-8 bits).++Rationale+---------+Both the classic GPTQ ``qweight`` layout and the AWQ ``qweight`` layout store+sub-byte integers inside ``int32`` words. The historical AutoRound+implementations special-cased ``bits in (2, 4, 8)`` (where ``32 % bits == 0``,+so every value fits neatly inside a single word) and ``bits == 3`` (a bespoke+hand-rolled routine). That is why 5/6/7-bit checkpoints could not be exported.++In reality *all* of those layouts are the exact same thing: a contiguous+little-endian bit-stream laid over blocks of **32 consecutive values**, which+always yields exactly ``bits`` ``int32`` words per block::++ value i (0 <= i < 32) occupies bits [i*bits, (i+1)*bits)+ word = (i * bits) // 32+ offset = (i * bits) % 32+ # when offset + bits > 32 the value straddles ``word`` and ``word + 1``++For ``bits in (2, 4, 8)`` no value ever straddles a word boundary, so this+degenerates to the familiar ``32 // bits`` values-per-word layout. For+``bits == 3`` it reproduces the legacy AutoRound/AutoGPTQ 3-bit routine+bit-for-bit (10 values, a straddling value, 10 values, ... per 3 words). And+for ``bits in (5, 6, 7)`` it "just works" -- no new concept is introduced.++This is also precisely the layout consumed by the `humming+<https://github.com/inclusionAI/humming>`_ GEMM kernels+(``humming/include/humming/kernel/pack_weight.cuh::common_pack_weight``),+which support any weight type below 8 bits. Sharing this single definition is+what lets ``auto_round:auto_gptq`` / ``auto_round:auto_awq`` checkpoints carry+5/6/7-bit weights and still be served by a fast kernel.++Only the block size (32 values) is required to divide the packed dimension;+there is no ``32 % bits == 0`` constraint.+"""++from typing import Optional++import torch++__all__ = [+ "AWQ_PACK_ORDER",+ "AWQ_REVERSE_ORDER",+ "SUPPORTED_PACKING_BITS",+ "awq_reverse_reorder",+ "awq_reorder",+ "pack_bitstream",+ "unpack_bitstream",+ "packed_dim_size",+ "requires_generic_bit_packing",+]++# Weight bit-widths that ``pack_bitstream`` / ``unpack_bitstream`` handle.+SUPPORTED_PACKING_BITS = (1, 2, 3, 4, 5, 6, 7, 8)++# Number of source values covered by one packing block. ``PACK_BLOCK * bits``+# is always a multiple of 32, so a block maps onto exactly ``bits`` int32 words.+PACK_BLOCK = 32++# AWQ shuffles values inside groups of 8 before writing them into the+# bit-stream. ``AWQ_PACK_ORDER`` maps *logical* position -> *stored* slot and+# ``AWQ_REVERSE_ORDER`` is its inverse (stored slot -> logical position).+# For 4-bit these reproduce AutoAWQ's classic ``[0, 4, 1, 5, 2, 6, 3, 7]``+# shift table exactly.+AWQ_INTERLEAVE = 8+AWQ_REVERSE_ORDER = (0, 4, 1, 5, 2, 6, 3, 7)+AWQ_PACK_ORDER = (0, 2, 4, 6, 1, 3, 5, 7)++_UINT32_MASK = 0xFFFFFFFF+_INT32_SIGN = 1 << 31+_UINT32_SPAN = 1 << 32+++def requires_generic_bit_packing(bits: int) -> bool:+ """Whether ``bits`` needs the generic bit-stream path.++ ``2/4/8`` (word-aligned) and ``3`` (legacy hand-rolled) keep their original+ fast paths so that previously exported checkpoints stay byte-identical.+ """+ return int(bits) in (5, 6, 7)+++def packed_dim_size(num_values: int, bits: int) -> int:+ """Number of ``int32`` words needed to store ``num_values`` values."""+ if num_values % PACK_BLOCK != 0:+ raise ValueError(+ f"cannot pack {num_values} values at {bits} bits: the packed dimension must be a multiple of "+ f"{PACK_BLOCK}. Pad the layer or leave it unquantized."+ )+ return num_values * bits // 32+++def _shift_tables(bits: int, device: torch.device):+ """Per-value (word index, bit offset) tables for one 32-value block."""+ idx = torch.arange(PACK_BLOCK, device=device, dtype=torch.int64) * bits+ word = torch.div(idx, 32, rounding_mode="floor")+ offset = idx % 32+ return word, offset+++def _check_bits(bits: int) -> int:+ bits = int(bits)+ if bits not in SUPPORTED_PACKING_BITS:+ raise ValueError(f"unsupported bits={bits} for bit-stream packing, expected one of {SUPPORTED_PACKING_BITS}")+ return bits+++def pack_bitstream(values: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:+ """Pack integer ``values`` into ``int32`` along ``dim`` using the bit-stream layout.++ Args:+ values: Integer tensor holding unsigned values in ``[0, 2**bits)``.+ bits: Weight bit-width, see :data:`SUPPORTED_PACKING_BITS`.+ dim: Dimension to pack. Its size must be a multiple of 32.++ Returns:+ ``int32`` tensor with ``dim`` shrunk to ``size * bits // 32``.+ """+ bits = _check_bits(bits)+ values = values.movedim(dim, -1)+ num_values = values.shape[-1]+ num_words = packed_dim_size(num_values, bits)++ lead_shape = values.shape[:-1]+ # (..., n_blocks, 32) — one block produces exactly ``bits`` words.+ values = values.reshape(*lead_shape, num_values // PACK_BLOCK, PACK_BLOCK).to(torch.int64)+ values = values & ((1 << bits) - 1)++ word, offset = _shift_tables(bits, values.device)+ packed = torch.zeros(*values.shape[:-1], bits, dtype=torch.int64, device=values.device)++ # Low part: the (possibly partial) chunk that lands in ``word``.+ # Bit ranges never overlap, so accumulating with ``index_add_`` is+ # equivalent to OR-ing but stays fully vectorized.+ packed.index_add_(-1, word, values << offset)++ straddles = (offset + bits) > 32+ if bool(straddles.any()):+ # High part: the remainder spills into the next word's low bits.+ high = torch.where(straddles, values >> (32 - offset), torch.zeros_like(values))+ packed.index_add_(-1, torch.clamp(word + 1, max=bits - 1), high)++ packed = packed & _UINT32_MASK+ # Reinterpret as signed int32 (torch has no uint32 storage).+ packed = torch.where(packed >= _INT32_SIGN, packed - _UINT32_SPAN, packed).to(torch.int32)+ packed = packed.reshape(*lead_shape, num_words)+ return packed.movedim(-1, dim)+++def unpack_bitstream(packed: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:+ """Inverse of :func:`pack_bitstream`.++ Returns an ``int32`` tensor of unsigned values in ``[0, 2**bits)`` with+ ``dim`` expanded back to ``num_words * 32 // bits``.+ """+ bits = _check_bits(bits)+ packed = packed.movedim(dim, -1)+ num_words = packed.shape[-1]+ if num_words % bits != 0:+ raise ValueError(f"packed dimension {num_words} is not a multiple of bits={bits}")++ lead_shape = packed.shape[:-1]+ packed = packed.reshape(*lead_shape, num_words // bits, bits).to(torch.int64) & _UINT32_MASK++ word, offset = _shift_tables(bits, packed.device)+ one = torch.ones((), dtype=torch.int64, device=packed.device)++ low_bits = torch.clamp(torch.full_like(offset, 32) - offset, max=bits)+ low = (packed.index_select(-1, word) >> offset) & (torch.bitwise_left_shift(one, low_bits) - 1)++ high_bits = torch.full_like(low_bits, bits) - low_bits+ if bool(torch.any(high_bits > 0)):+ next_word = packed.index_select(-1, torch.clamp(word + 1, max=bits - 1))+ high = torch.bitwise_left_shift(next_word & (torch.bitwise_left_shift(one, high_bits) - 1), low_bits)+ low = low | torch.where(high_bits > 0, high, torch.zeros_like(high))++ values = low.reshape(*lead_shape, num_words * 32 // bits).to(torch.int32)+ return values.movedim(-1, dim)+++def _apply_awq_order(tensor: torch.Tensor, order, dim: int) -> torch.Tensor:+ tensor = tensor.movedim(dim, -1)+ size = tensor.shape[-1]+ if size % AWQ_INTERLEAVE != 0:+ raise ValueError(+ f"AWQ interleaving requires the packed dimension ({size}) to be a multiple of {AWQ_INTERLEAVE}"+ )+ index = torch.as_tensor(order, dtype=torch.long, device=tensor.device)+ tensor = tensor.reshape(*tensor.shape[:-1], size // AWQ_INTERLEAVE, AWQ_INTERLEAVE)+ tensor = tensor.index_select(-1, index)+ tensor = tensor.reshape(*tensor.shape[:-2], size)+ return tensor.movedim(-1, dim)+++def awq_reorder(values: torch.Tensor, dim: int = -1) -> torch.Tensor:+ """Shuffle logical values into AWQ storage order (apply before packing)."""+ return _apply_awq_order(values, AWQ_PACK_ORDER, dim)+++def awq_reverse_reorder(values: torch.Tensor, dim: int = -1) -> torch.Tensor:+ """Undo :func:`awq_reorder` (apply after unpacking)."""+ return _apply_awq_order(values, AWQ_REVERSE_ORDER, dim)+++def pack_awq_bitstream(values: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:+ """AWQ-flavoured packing: interleave groups of 8, then bit-stream pack."""+ return pack_bitstream(awq_reorder(values, dim=dim), bits, dim=dim)+++def unpack_awq_bitstream(packed: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:+ """Inverse of :func:`pack_awq_bitstream`."""+ return awq_reverse_reorder(unpack_bitstream(packed, bits, dim=dim), dim=dim)+++def pack_scalar_zero(value: int, bits: int, num_words: int, shape, device=None) -> torch.Tensor:+ """Build a ``qzeros`` tensor where every group shares the same zero point.++ ``value`` is broadcast to ``PACK_BLOCK`` entries and packed once, then the+ resulting ``bits``-word pattern is tiled across ``num_words`` columns.+ """+ bits = _check_bits(bits)+ block = torch.full((PACK_BLOCK,), int(value), dtype=torch.int64, device=device)+ pattern = pack_bitstream(block, bits) # (bits,)+ repeats = (num_words + bits - 1) // bits+ row = pattern.repeat(repeats)[:num_words]+ return row.reshape(*(1,) * (len(shape) - 1), num_words).expand(*shape).contiguous()+++def infer_packed_bits(num_values: int, num_words: int) -> Optional[int]:+ """Best-effort recovery of ``bits`` from packed/unpacked dimension sizes."""+ if num_values <= 0 or num_words <= 0:+ return None+ bits = num_words * 32 // num_values+ return bits if bits in SUPPORTED_PACKING_BITS and num_values * bits == num_words * 32 else Nonediff --git a/auto_round_extension/humming/__init__.py b/auto_round_extension/humming/__init__.pynew file mode 100644index 000000000..1a48c8a45--- /dev/null+++ b/auto_round_extension/humming/__init__.py@@ -0,0 +1,22 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round_extension.humming.qlinear_humming import (+ QuantLinear,+ QuantLinearAWQ,+ QuantLinearGPTQ,+ is_humming_available,+)++__all__ = ["QuantLinear", "QuantLinearAWQ", "QuantLinearGPTQ", "is_humming_available"]diff --git a/auto_round_extension/humming/qlinear_humming.py b/auto_round_extension/humming/qlinear_humming.pynew file mode 100644index 000000000..421920f8e--- /dev/null+++ b/auto_round_extension/humming/qlinear_humming.py@@ -0,0 +1,234 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Inference backend backed by the `humming <https://github.com/inclusionAI/humming>`_ GEMM kernels.++humming is a JIT-compiled quantized-GEMM library that supports **any** weight+bit-width from 1 to 8 bits (SM75+), which makes it the natural kernel for+AutoRound's 5/6/7-bit checkpoints -- the Triton/torch fallbacks either reject+those bit-widths or dequantize them in Python.++The checkpoint layouts AutoRound writes map one-to-one onto humming's schemas:++=============================== ================================= ==================+packing_format humming schema zero point+=============================== ================================= ==================+``auto_round`` / ``:gptqmodel`` ``GPTQWeightSchema`` stored as-is+``auto_round:auto_gptq`` ``GPTQWeightSchema`` ``zp - 1`` (sym only)+``auto_round:auto_awq`` ``AWQWeightSchema`` stored as-is+=============================== ================================= ==================++For symmetric quantization humming ignores ``qzeros`` entirely and implicitly+dequantizes with ``(value - 2**(bits-1)) * scale`` (see+``humming/utils/weight.py::dequantize_weight``), which is exactly AutoRound's+full-range-symmetric convention -- so both the ``zp`` and ``zp - 1`` GPTQ+variants land on the same (correct) numerics.+"""++import math+from logging import getLogger++import torch+import torch.nn as nn++logger = getLogger(__name__)++# humming ships only as a git repo (distribution name ``humming-kernels``).+HUMMING_INSTALL_SPEC = "git+https://github.com/inclusionAI/humming.git"++# humming's weight transform asserts ``padded_shape_n % 64 == 0`` and+# ``padded_shape_k % (2 * 256 // act_bits) == 0`` (32 for fp16/bf16 activations).+PAD_N_TO_MULTIPLE = 64+PAD_K_TO_MULTIPLE = 32++SUPPORTED_BITS = (2, 3, 4, 5, 6, 7, 8)+++def is_humming_available() -> bool:+ """Whether the humming kernels can be imported and used on this machine."""+ try:+ import humming # noqa: F401 # pylint: disable=E0401+ except Exception: # pragma: no cover - depends on the local environment+ return False+ return torch.cuda.is_available()+++class QuantLinear(nn.Module):+ """Weight-only quantized linear served by humming.++ Buffers are registered with the *checkpoint* layout so that HuggingFace's+ state-dict loader can fill them directly. ``post_init()`` then hands them to+ humming, which repacks them into its kernel-native layout and releases the+ originals.+ """++ QUANT_TYPE = "humming"+ # packing_format string handed to humming's AutoRoundWeightSchema+ PACKING_FORMAT = "auto_round"+ # AWQ stores qweight transposed relative to GPTQ+ AWQ_LAYOUT = False++ def __init__(+ self,+ bits,+ group_size,+ infeatures,+ outfeatures,+ bias,+ sym=True,+ weight_dtype=torch.float16,+ **kwargs,+ ):+ super().__init__()+ if bits not in SUPPORTED_BITS:+ raise NotImplementedError(f"humming backend supports {SUPPORTED_BITS} bits, got {bits}")+ if infeatures % 32 != 0:+ raise ValueError(f"humming requires in_features ({infeatures}) to be a multiple of 32")+ if outfeatures % 32 != 0:+ raise ValueError(f"humming requires out_features ({outfeatures}) to be a multiple of 32")++ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.sym = bool(sym)+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1+ self.compute_dtype = weight_dtype if weight_dtype in (torch.float16, torch.bfloat16) else torch.float16++ num_groups = math.ceil(infeatures / self.group_size)+ packed_out = outfeatures * self.bits // 32++ if self.AWQ_LAYOUT:+ qweight_shape = (infeatures, packed_out)+ else:+ qweight_shape = (infeatures * self.bits // 32, outfeatures)++ self.register_buffer("qweight", torch.zeros(qweight_shape, dtype=torch.int32))+ self.register_buffer("qzeros", torch.zeros((num_groups, packed_out), dtype=torch.int32))+ self.register_buffer("scales", torch.zeros((num_groups, outfeatures), dtype=self.compute_dtype))+ if bias:+ self.register_buffer("bias", torch.zeros((outfeatures,), dtype=self.compute_dtype))+ else:+ self.bias = None++ self.humming_layer = None++ # ------------------------------------------------------------------ #+ # setup+ # ------------------------------------------------------------------ #+ def _build_schema(self):+ from humming.schema.autoround import AutoRoundWeightSchema # pylint: disable=E0401++ return AutoRoundWeightSchema(+ bits=self.bits,+ group_size=self.group_size,+ sym=self.sym,+ data_type="int",+ packing_format=self.PACKING_FORMAT,+ act_bits=16,+ )++ def post_init(self):+ """Hand the loaded checkpoint buffers to humming and repack them."""+ if self.humming_layer is not None:+ return++ try:+ from humming.layer import HummingLayer # pylint: disable=E0401+ except Exception as e: # pragma: no cover - depends on the local environment+ raise ImportError(+ "The humming kernels are required for this backend "+ "(and are the only real kernels for 5/6/7-bit weights). Install them with:\n"+ f" pip install {HUMMING_INSTALL_SPEC}"+ ) from e++ device = self.qweight.device+ if device.type != "cuda":+ raise RuntimeError(f"humming kernels require a CUDA device, but the layer lives on {device}")++ layer = HummingLayer(+ shape_n=self.outfeatures,+ shape_k=self.infeatures,+ weight_config=self._build_schema(),+ has_bias=self.bias is not None,+ pad_n_to_multiple=PAD_N_TO_MULTIPLE,+ pad_k_to_multiple=PAD_K_TO_MULTIPLE,+ torch_dtype=self.compute_dtype,+ )++ tensors = {+ "qweight": self.qweight,+ "qzeros": self.qzeros,+ "scales": self.scales.to(self.compute_dtype),+ }+ if self.bias is not None:+ tensors["bias"] = self.bias.to(self.compute_dtype)+ layer.load_from_tensors(tensors)+ layer = layer.to(device)+ # JIT-compiles / selects the kernel and rewrites the weights in place.+ layer.transform()++ # Release the checkpoint-layout buffers; humming owns the weights now.+ for name in ("qweight", "qzeros", "scales", "bias"):+ if name in self._buffers:+ self._buffers.pop(name)+ self.bias = None+ self.humming_layer = layer++ def pack(self, linear, scales, zeros, g_idx=None, device=None):+ raise NotImplementedError(+ "the humming backend is inference-only; export with the `auto_round`, "+ "`auto_round:auto_gptq` or `auto_round:auto_awq` format instead"+ )++ # ------------------------------------------------------------------ #+ # forward+ # ------------------------------------------------------------------ #+ def forward(self, x):+ if self.humming_layer is None:+ self.post_init()++ out_shape = x.shape[:-1] + (self.outfeatures,)+ x_dtype = x.dtype+ x = x.reshape(-1, x.shape[-1]).to(self.compute_dtype)+ out = self.humming_layer(x)+ # humming may pad the N dimension up to a multiple of 64.+ if out.shape[-1] != self.outfeatures:+ out = out[..., : self.outfeatures]+ return out.reshape(out_shape).to(x_dtype)+++class QuantLinearGPTQ(QuantLinear):+ """``auto_round:auto_gptq`` packing (``qzeros`` stores ``zp - 1``)."""++ QUANT_TYPE = "humming_gptq"+ PACKING_FORMAT = "auto_round:auto_gptq"+ AWQ_LAYOUT = False+++class QuantLinearAWQ(QuantLinear):+ """``auto_round:auto_awq`` packing (interleaved, transposed qweight)."""++ QUANT_TYPE = "humming_awq"+ PACKING_FORMAT = "auto_round:auto_awq"+ AWQ_LAYOUT = True+++__all__ = [+ "QuantLinear",+ "QuantLinearAWQ",+ "QuantLinearGPTQ",+ "SUPPORTED_BITS",+ "is_humming_available",+]diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 0c6bd0d58..466057700 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -21,9 +21,18 @@ import torch.nn as nnimport transformersfrom auto_round.utils import get_packing_device+from auto_round.utils.bit_packing import (+ pack_bitstream,+ pack_scalar_zero,+ requires_generic_bit_packing,+ unpack_bitstream,+)logger = getLogger(__name__)+# Bit-widths this QuantLinear can pack/unpack. 2/4/8 are word-aligned, 3 uses the+# legacy hand-rolled routine, and 5/6/7 go through the generic bit-stream helpers.+SUPPORTED_BITS = (2, 3, 4, 5, 6, 7, 8)_DEVICE_WF_3BITS_TENSORS = {}# Constants for FP4 values (E2M1 format)@@ -44,7 +53,62 @@ def get_wf_3bits_tensor(device):return _DEVICE_WF_3BITS_TENSORS[device_str]-class QuantLinear(nn.Module):+class GenericBitPackingMixin:+ """Shared 5/6/7-bit packing for the GPTQ-style QuantLinear variants.++ Both ``qlinear_torch`` (raw zero point) and ``qlinear_torch_zp`` (GPTQ's+ ``zp - 1`` convention) produce the same bit layout, differing only by a+ constant offset applied to the zero points.+ """++ def _to_intweight(self, linear, scales, zeros, device):+ """Dequantization-domain -> integer grid, shared by the packing routines."""+ W = linear.weight.data.to(device).clone()+ if type(linear) == nn.Conv2d:+ W = W.flatten(1)+ if type(linear) == transformers.pytorch_utils.Conv1D:+ W = W.t()++ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])+ else:+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + zeros)+ return intweight.to(torch.int32)++ def pack_generic_bits(self, linear, scales, zeros, g_idx=None, device=None, zp_offset=0):+ """Pack 5/6/7-bit weights using the generic little-endian bit-stream layout.++ The resulting ``qweight``/``qzeros`` follow exactly the same convention as+ the 2/3/4/8-bit paths (values laid out contiguously over blocks of 32+ along the packed dimension), so downstream kernels -- including humming --+ can consume every bit-width with one code path.++ ``zp_offset`` is ``-1`` for the GPTQ-style "zero point minus one" variant.+ """+ device = get_packing_device(device)+ scales_t = scales.t().contiguous()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()++ intweight = self._to_intweight(linear, scales, zeros, device)+ # (out, in) -> pack along ``in`` -> transpose to the (in*bits/32, out) layout+ self.qweight = pack_bitstream(intweight, self.bits, dim=1).t().contiguous().to("cpu")+ del intweight++ num_zero_words = scales_t.shape[1] * self.bits // 32+ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous().to(device).to(torch.int32) + zp_offset+ self.qzeros = pack_bitstream(zeros, self.bits, dim=1).to("cpu")+ else:+ self.qzeros = pack_scalar_zero(+ int(zeros) + zp_offset, self.bits, num_zero_words, (scales_t.shape[0], num_zero_words)+ ).to("cpu")+++class QuantLinear(GenericBitPackingMixin, nn.Module):"""Torch quantized linear layer."""@@ -53,8 +117,8 @@ class QuantLinear(nn.Module):def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):super().__init__()- if bits not in [2, 3, 4, 8]:- raise NotImplementedError("Only 2,3,4,8 bits are supported.")+ if bits not in SUPPORTED_BITS:+ raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")self.infeatures = infeaturesself.outfeatures = outfeaturesself.bits = bits@@ -95,13 +159,18 @@ class QuantLinear(nn.Module):self.trainable = trainable# is performed by unpacking the weights and using torch.matmul- if self.bits in [2, 4, 8]:+ self.use_generic_bit_packing = requires_generic_bit_packing(self.bits)+ if self.use_generic_bit_packing:+ # 5/6/7-bit values straddle int32 word boundaries, so a single shift+ # table cannot describe them; unpacking goes through the generic helper.+ self.wf = None+ elif self.bits in [2, 4, 8]:self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device).unsqueeze(0)elif self.bits == 3:self.wf = get_wf_3bits_tensor(device=self.qweight.device)- self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8+ self.dequant_dtype = torch.int16 if self.bits > 7 else torch.int8def post_init(self):pass@@ -291,15 +360,21 @@ class QuantLinear(nn.Module):return self.pack_248_bits(linear, scales, zeros, g_idx, device)elif self.bits in [3]:return self.pack_3bits(linear, scales, zeros, g_idx, device)+ elif self.use_generic_bit_packing:+ return self.pack_generic_bits(linear, scales, zeros, g_idx, device)else:- raise ValueError("Only 2,3,4,8 bits are supported.")+ raise ValueError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)x = x.reshape(-1, x.shape[-1])x_dtype = x.dtype- if self.bits in [2, 4, 8]:+ if self.use_generic_bit_packing:+ # Generic bit-stream layout: unpack straight into 2-D tensors.+ zeros = unpack_bitstream(self.qzeros, self.bits, dim=1).reshape(self.scales.shape)+ weight = unpack_bitstream(self.qweight, self.bits, dim=0)+ elif self.bits in [2, 4, 8]:if self.wf.device != self.qzeros.device:self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device@@ -341,7 +416,8 @@ class QuantLinear(nn.Module):weight = weight & 0x7weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)- weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])+ if weight.dim() == 3:+ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])if hasattr(self, "g_idx"):num_itr = self.g_idx.shape[0] // x.shape[-1]num_dim = self.g_idx.shape[0] // num_itrdiff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex 5da317b71..d6fef7beb 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -21,12 +21,17 @@ import torch.nn as nnimport transformersfrom auto_round.utils import get_packing_device-from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensor+from auto_round.utils.bit_packing import requires_generic_bit_packing, unpack_bitstream+from auto_round_extension.torch.qlinear_torch import (+ SUPPORTED_BITS,+ GenericBitPackingMixin,+ get_wf_3bits_tensor,+)logger = getLogger(__name__)-class QuantLinear(nn.Module):+class QuantLinear(GenericBitPackingMixin, nn.Module):"""Torch quantized linear layer."""@@ -35,8 +40,8 @@ class QuantLinear(nn.Module):def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):super().__init__()- if bits not in [2, 3, 4, 8]:- raise NotImplementedError("Only 2,3,4,8 bits are supported.")+ if bits not in SUPPORTED_BITS:+ raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")self.infeatures = infeaturesself.outfeatures = outfeaturesself.bits = bits@@ -77,14 +82,18 @@ class QuantLinear(nn.Module):self.trainable = trainable# is performed by unpacking the weights and using torch.matmul- if self.bits in [2, 4, 8]:+ self.use_generic_bit_packing = requires_generic_bit_packing(self.bits)+ if self.use_generic_bit_packing:+ # 5/6/7-bit values straddle int32 word boundaries; use the generic helper.+ self.wf = None+ elif self.bits in [2, 4, 8]:self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device).unsqueeze(0)else: ## bits == 3self.wf = get_wf_3bits_tensor(device=self.qweight.device)- self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8+ self.dequant_dtype = torch.int16 if self.bits > 7 else torch.int8def post_init(self):pass@@ -288,15 +297,22 @@ class QuantLinear(nn.Module):return self.pack_248_bits(linear, scales, zeros, g_idx, device)elif self.bits in [3]:return self.pack_3bits(linear, scales, zeros, g_idx, device)+ elif self.use_generic_bit_packing:+ # GPTQ stores ``zp - 1``; reuse the shared generic packer with that offset.+ return self.pack_generic_bits(linear, scales, zeros, g_idx, device, zp_offset=-1)else:- raise ValueError("Only 2,3,4,8 bits are supported.")+ raise ValueError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)x = x.reshape(-1, x.shape[-1])x_dtype = x.dtype- if self.bits in [2, 4, 8]:+ if self.use_generic_bit_packing:+ # Generic bit-stream layout: unpack straight into 2-D tensors.+ zeros = unpack_bitstream(self.qzeros, self.bits, dim=1).reshape(self.scales.shape)+ weight = unpack_bitstream(self.qweight, self.bits, dim=0)+ elif self.bits in [2, 4, 8]:if self.wf.device != self.qzeros.device:self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device@@ -338,7 +354,8 @@ class QuantLinear(nn.Module):weight = weight & 0x7weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)zeros += 1- weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])+ if weight.dim() == 3:+ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])if hasattr(self, "g_idx"):num_itr = self.g_idx.shape[0] // x.shape[-1]num_dim = self.g_idx.shape[0] // num_itrdiff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 03a5614af..682f2bdb2 100755--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -123,9 +123,9 @@ AutoRound supports several Schemes:- **W4A16**(bits:4,group_size:128,sym:True,act_bits:16)- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)-- **W7A16**(bits:7,group_size:128,sym:True,act_bits:16) - `llm_compressor` format-- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats-- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats+- **W7A16**(bits:7,group_size:128,sym:True,act_bits:16)+- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16)+- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16)- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)- **Group-size variants** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor` (W2-W8); `auto_round` / `auto_gptq` (W2 only)@@ -146,7 +146,17 @@ Besides, you could modify the `group_size`, `bits`, `sym` and many other configsYou can use command `auto_round list format` to show all supported formats with support scheme.**AutoRound Format**: This format is well-suited for CPU, Intel GPU, CUDA and HPU devices, 2 bits, as well as mixed-precision-inference. **[2,3,4,8] bits are supported**. Please set `--format auto_round`+inference. **[2,3,4,5,6,7,8] bits are supported**. Please set `--format auto_round`++> **About 5/6/7 bits**: these widths do not divide 32 evenly, so each value is stored in a contiguous+> little-endian bit-stream laid over blocks of 32 values (`in_features * bits / 32` int32 words).+> This is a strict generalization of the existing layouts — 2/4/8-bit and 3-bit checkpoints keep+> byte-identical `qweight`/`qzeros`. 5/6/7 bits work with `auto_round`, `auto_round:auto_gptq` and+> `auto_round:auto_awq`, and can be freely combined in mixed-bit recipes. Upstream AutoGPTQ/AutoAWQ+> kernels cannot read them, so the plain `auto_gptq` / `auto_awq` formats still reject 5/6/7 bits.+> Both `in_features` and `out_features` must be multiples of 32. For inference, install the+> [humming](https://github.com/inclusionAI/humming) kernels (`pip install git+https://github.com/inclusionAI/humming.git`);+> otherwise AutoRound falls back to the slow `torch` dequantize-and-matmul backend.**GGUF** Format: Experimental feature. This format is well-suited for CPU devices and is widely adopted by thecommunity. `q*_k`,`q*_0`,`q*_1` are supported. Please set `--format gguf:q4_k_m`, `--format gguf:q2_k_s`, etc@@ -158,6 +168,7 @@ models. Besides, recently 3 bits may have some accuracy issues in Transformers.**AutoAWQ Format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widelyadopted within the community, **only 4-bits quantization is supported**. Please set `--format auto_awq`+(the AutoRound-flavoured `auto_round:auto_awq` additionally supports 5/6/7 bits).**LLM-Compressor Format**: **NVFP4, MXFP4(kernel in WIP), MXFP8 are supported**. Please set `--format llm_compressor`@@ -171,10 +182,12 @@ adopted within the community, **only 4-bits quantization is supported**. Please| Format | Supported Schemes ||:-----------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16`, `MXINT4` |+| **auto_round** | W4A16, W2A16, W3A16, W5A16, W6A16, W7A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16`, `MXINT4` || **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK, W2A16, W3A16, W4A16, W5A16, W6A16, W7A16, W8A16 and G64/G32 variants of W2-W8 || **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 || **mlx** / **auto_round:mlx** | W2A16, W3A16, W4A16, W5A16, W6A16, W8A16, BF16, mixed-bit / mixed-group_size (Apple Silicon only) |+| **auto_round:auto_awq** | W4A16, W5A16, W6A16, W7A16, BF16 |+| **auto_round:auto_gptq** | W4A16, W2A16, W3A16, W5A16, W6A16, W7A16, W8A16, W2A16G64, W2A16G32, BF16 || **auto_awq** | W4A16, BF16 || **auto_gptq** | W4A16, W2A16, W3A16, W8A16,W2A16G64, W2A16G32, BF16 || **fp8** | FP8_BLOCK |@@ -489,7 +502,7 @@ auto_round \#### API Usage~~~avg_bits= 3.0-scheme = AutoScheme(avg_bits=avg_bits, options=("W2A16G64“, "W4A16","W8A16"))+scheme = AutoScheme(avg_bits=avg_bits, options=("W2A16G64", "W4A16", "W8A16"))ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)ar.quantize_and_save()~~~@@ -1019,7 +1032,12 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal### CUDA-Supports 2, 3, 4, and 8 bits. We recommend using GPTQModel for 4 and 8 bits inference.+Supports 2, 3, 4, 5, 6, 7 and 8 bits. We recommend using GPTQModel for 4 and 8 bits inference, and+[humming](https://github.com/inclusionAI/humming) for 5/6/7 bits (it is also faster than Triton for the other widths).++```bash+pip install git+https://github.com/inclusionAI/humming.git+``````pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -1088,8 +1106,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| gptqmodel:awq_marlin | cuda | 4,8 | FP16 | 5 | awq | gptqmodel || gptqmodel:awq_gemm | cuda | 4 | FP16 | 3 | awq | gptqmodel || gptqmodel:awq_torch | cuda/cpu | 4 | FP16 | 2 | awq | gptqmodel |+| humming | cuda | 2,3,4,5,6,7,8 | BF16/FP16 | 4 | gptq/gptq_zp+-1 | humming-kernels |+| humming | cuda | 2,3,4,5,6,7,8 | BF16/FP16 | 4 | awq | humming-kernels || hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |-| torch | xpu/cpu/cuda | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |+| torch | xpu/cpu/cuda | 2,3,4,5,6,7,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |### Convert GPTQ/AWQ to AutoRounddiff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 7d0f3d5de..9a0563822 100755--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -117,9 +117,9 @@ pip install auto-roundAutoRound 支持多种量化配置:- **W4A16**(bits:4, group_size:128, sym:True, act_bits:16) # 4位权重,分组大小为128,对称量化,16位激活,- **W8A16**(bits:8, group_size:128, sym:True, act_bits:16)-- **W7A16**(bits:7, group_size:128, sym:True, act_bits:16) - 仅 `llm_compressor` 格式-- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式-- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式+- **W7A16**(bits:7, group_size:128, sym:True, act_bits:16)+- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16)+- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16)- **W3A16**(bits:3, group_size:128, sym:True, act_bits:16)- **W2A16**(bits:2, group_size:128, sym:True, act_bits:16)- **分组大小变体** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor`(W2-W8);`auto_round` / `auto_gptq`(仅 W2)@@ -141,13 +141,21 @@ AutoRound 支持多种量化配置:执行 `auto_round list format` 可查看所有支持的导出格式及其对应的量化方案。-**AutoRound 原生格式**:适用于 CPU、英特尔 GPU、CUDA、HPU 等设备,支持2位宽及混合精度推理,**兼容 [2、3、4、8] bits**。使用时需设置 `--format auto_round`。+**AutoRound 原生格式**:适用于 CPU、英特尔 GPU、CUDA、HPU 等设备,支持2位宽及混合精度推理,**兼容 [2、3、4、5、6、7、8] bits**。使用时需设置 `--format auto_round`。++> **关于 5/6/7 bits**:这几种位宽无法整除 32,因此每个值被存放在以 32 个元素为一组、按小端连续排布的比特流中+> (共 `in_features * bits / 32` 个 int32)。这是对现有排布的严格推广——2/4/8 bit 与 3 bit 的+> `qweight`/`qzeros` 与之前完全逐字节一致。5/6/7 bits 可用于 `auto_round`、`auto_round:auto_gptq`+> 和 `auto_round:auto_awq`,也可以自由用于混合 bit 配方。上游 AutoGPTQ/AutoAWQ 的 kernel 无法读取该排布,+> 因此原生的 `auto_gptq` / `auto_awq` 格式仍然拒绝 5/6/7 bits。`in_features` 与 `out_features` 都必须是 32 的倍数。+> 推理时请安装 [humming](https://github.com/inclusionAI/humming) kernel+> (`pip install git+https://github.com/inclusionAI/humming.git`),否则会回退到较慢的 `torch` 反量化后再矩乘的后端。**GGUF 格式**:实验性功能,适用于 CPU 设备,是社区主流格式之一,支持 `q*_k`、`q*_0`、`q*_1` 系列的量化。需设置 `--format gguf:q4_k_m`、`--format gguf:q2_k_s`等具体格式。**AutoGPTQ 格式**:适用于 CUDA 设备的对称量化,在社区中广泛应用,**兼容 [2、3、4、8] bits **(但其**非对称推理核存在问题**,可能导致模型的精度大幅下降,尤其是在 2-bit 量化和小模型的场景;近期 Transformers 框架中 3-bits 量化也存在类似问题)。配置时需设置 `--format auto_gptq`。-**AutoAWQ 格式**:适用于 CUDA 设备的 4 位非对称量化,在社区中也广泛应用。**仅支持 4-bit 量化**。需设置 `--format auto_awq`。+**AutoAWQ 格式**:适用于 CUDA 设备的 4 位非对称量化,在社区中也广泛应用。**仅支持 4-bit 量化**。需设置 `--format auto_awq`(AutoRound 风格的 `auto_round:auto_awq` 额外支持 5/6/7 bits)。**LLM-Compressor 格式**:**支持 NVFP4、MXFP4(kernel 开发中)、MXFP8** 等。需设置 `--format llm_compressor`。@@ -163,9 +171,11 @@ AutoRound 支持多种量化配置:| 格式 | 支持的量化方案 ||:-------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------|-| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` |+| **auto_round** | W4A16、W2A16、W3A16、W5A16、W6A16、W7A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` || **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 || **llm_compressor** | NVFP4、W2A16、W3A16、W4A16、W5A16、W6A16、W7A16、W8A16 及 W2-W8 的 G64/G32 变体、FP8_BLOCK、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` |+| **auto_round:auto_awq** | W4A16、W5A16、W6A16、W7A16、BF16 |+| **auto_round:auto_gptq** | W4A16、W2A16、W3A16、W5A16、W6A16、W7A16、W8A16、W2A16G64、W2A16G32、BF16 || **auto_awq** | W4A16、BF16 || **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 || **mlx** / **auto_round:mlx** (实验性功能) | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) |@@ -977,7 +987,14 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal```### CUDA-支持 2、3、4、8 bits 量化模型,**4-bits/8-bits 推理推荐使用 GPTQModel 后端**,使用示例:+支持 2、3、4、5、6、7、8 bits 量化模型,**4-bits/8-bits 推理推荐使用 GPTQModel 后端**;**5/6/7 bits 推荐使用+[humming](https://github.com/inclusionAI/humming) 后端**(它在其他位宽上也比 Triton 更快):++```bash+pip install git+https://github.com/inclusionAI/humming.git+```++使用示例:```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -1049,8 +1066,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| gptqmodel:awq_marlin | cuda | 4、8 | FP16 | 5 | awq | gptqmodel || gptqmodel:awq_gemm | cuda | 4 | FP16 | 3 | awq | gptqmodel || gptqmodel:awq_torch | cuda/cpu | 4 | FP16 | 2 | awq | gptqmodel |+| humming | cuda | 2、3、4、5、6、7、8 | BF16/FP16 | 4 | gptq/gptq_zp+-1 | humming-kernels |+| humming | cuda | 2、3、4、5、6、7、8 | BF16/FP16 | 4 | awq | humming-kernels || hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |-| torch | xpu/cpu/cuda | 2、3、4、8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |+| torch | xpu/cpu/cuda | 2、3、4、5、6、7、8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |### 将 GPTQ 或 AWQ 模型转换为 AutoRound 格式为了提升兼容性(尤其是英特尔设备),大部分 GPTQ/AWQ 量化模型均可转换为 AutoRound 格式。**注意**:若模型再次存储,其量化配置可能会发生变更, 由 gptq/awq 量化变化成 auto-round 量化。diff --git a/test/unit/test_cpu/export/test_export.py b/test/unit/test_cpu/export/test_export.pyindex 0822782b9..e2310fab0 100644--- a/test/unit/test_cpu/export/test_export.py+++ b/test/unit/test_cpu/export/test_export.py@@ -556,7 +556,10 @@ class TestAutoRound:sym=True,)ar.post_init()- with pytest.raises(ValueError, match="auto_awq format support quantization scheme with W4A16 but got bits=2"):+ with pytest.raises(+ ValueError,+ match="auto_awq format support quantization scheme with W4A16,W5A16,W6A16,W7A16 but got bits=2",+ ):resolve_formats(ResolvedScheme.from_scheme(ar.scheme_context),format="auto_round:auto_awq",