Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support 5,6,7 bits on cuda and add humming backend (#2281)

Wenhua Cheng1f23576afd25
diff --git a/auto_round/compressors/mllm/dataset.py b/auto_round/compressors/mllm/dataset.py
index 572227655..fec57f44a 100644
--- a/auto_round/compressors/mllm/dataset.py
+++ b/auto_round/compressors/mllm/dataset.py
@@ -56,12 +56,21 @@ def register_dataset(name_list):
class LlavaDataset(Dataset):
"""Dataset for supervised fine-tuning."""
+ LLAVA_REPO_ID = "liuhaotian/LLaVA-Instruct-150K"
BASE_LLAVA_URL = "https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/"
+ LLAVA_DATASET_FILES = {
+ "llava_conv_58k": "conversation_58k.json",
+ "llava_instruct_80k": "llava_instruct_80k.json",
+ "llava_instruct_150k": "llava_instruct_150k.json",
+ }
+ # Kept for backward compatibility, `LLAVA_DATASET_FILES` is the source of truth.
LLAVA_DATASET = {
"llava_conv_58k": BASE_LLAVA_URL + "conversation_58k.json?download=true",
"llava_instruct_80k": BASE_LLAVA_URL + "llava_instruct_80k.json?download=true",
"llava_instruct_150k": BASE_LLAVA_URL + "llava_instruct_150k.json?download=true",
}
+ DOWNLOAD_TIMEOUT = 60
+ DOWNLOAD_RETRIES = 3
_COCO_DATA_URL = "http://images.cocodataset.org/train2017/"
IMAGE_TOKEN = "<image>"
MAX_SUPPORT_SEQLEN = 512
@@ -88,18 +97,18 @@ class LlavaDataset(Dataset):
logger.info(f"use dataset {dataset_path}, loading from disk...")
self.questions = json.load(open(dataset_path, "r"))
else:
- import requests
-
if dataset_path == "liuhaotian/llava":
dataset_path = "llava_conv_58k"
else:
dataset_path = dataset_path.split("/")[-1]
dataset_name = dataset_path.split("/")[-1]
- if dataset_name in self.LLAVA_DATASET:
+ if dataset_name in self.LLAVA_DATASET_FILES:
logger.info(f"use dataset {dataset_name}, downloading...")
- self.questions = requests.get(self.LLAVA_DATASET[dataset_name], stream=True).json()
+ self.questions = self._download_questions(dataset_name)
else:
- raise KeyError(f"{dataset_path} is not support, we support {self.LLAVA_DATASET.keys()}.")
+ raise KeyError(
+ f"{dataset_path} is not supported; supported datasets: {', '.join(self.LLAVA_DATASET_FILES.keys())}."
+ )
self.seqlen = seqlen
self.questions = self.check(self.questions, self.seqlen, nsamples)
@@ -117,6 +126,56 @@ class LlavaDataset(Dataset):
image_fold = image_fold["image"]
self.image_fold = image_fold
+ @classmethod
+ def _download_questions(cls, dataset_name: str):
+ """Download a LLaVA instruction dataset and return the decoded JSON payload.
+
+ ``huggingface_hub`` is preferred because it honours ``HF_TOKEN``/``HF_ENDPOINT`` mirrors,
+ retries transient failures and caches the file locally. A plain HTTP request is used as a
+ fallback. Both paths validate the payload, so a non-JSON body (an HTML error/rate-limit
+ page, an empty response from a proxy, ...) raises an actionable error instead of a bare
+ ``JSONDecodeError``.
+ """
+ filename = cls.LLAVA_DATASET_FILES[dataset_name]
+ errors = []
+
+ try:
+ from huggingface_hub import hf_hub_download
+
+ local_path = hf_hub_download(repo_id=cls.LLAVA_REPO_ID, filename=filename, repo_type="dataset")
+ with open(local_path, encoding="utf-8") as f:
+ return json.load(f)
+ except Exception as e:
+ errors.append(f"huggingface_hub: {type(e).__name__}: {e}")
+ logger.warning(
+ f"failed to fetch dataset '{dataset_name}' via huggingface_hub ({type(e).__name__}: {e}),"
+ " falling back to a direct http download."
+ )
+
+ import requests
+
+ url = cls.LLAVA_DATASET[dataset_name]
+ for attempt in range(1, cls.DOWNLOAD_RETRIES + 1):
+ try:
+ response = requests.get(url, timeout=cls.DOWNLOAD_TIMEOUT)
+ response.raise_for_status()
+ return json.loads(response.content.decode("utf-8"))
+ except Exception as e:
+ errors.append(f"http attempt {attempt}: {type(e).__name__}: {e}")
+ if attempt < cls.DOWNLOAD_RETRIES:
+ logger.warning(
+ f"failed to download dataset '{dataset_name}' from {url}"
+ f" ({type(e).__name__}: {e}), retrying (attempt {attempt + 1}/{cls.DOWNLOAD_RETRIES})..."
+ )
+
+ details = "\n ".join(errors)
+ raise RuntimeError(
+ f"could not download the calibration dataset '{dataset_name}' from"
+ f" '{cls.LLAVA_REPO_ID}'. Please check your network connection/proxy, or download"
+ f" '{filename}' manually and pass its local path via the `dataset` argument."
+ f"\nCauses:\n {details}"
+ )
+
def check(self, questions, word_len, nsamples):
def _check(questions, min_word_len, max_word_len, nsamples):
new_questions = []
diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.py
index 86db8a3ee..662987529 100644
--- a/auto_round/export/export_to_awq/utils.py
+++ b/auto_round/export/export_to_awq/utils.py
@@ -40,6 +40,18 @@ import torch.nn as nn
from torch.autograd import Function
from auto_round.utils import get_packing_device
+from auto_round.utils.bit_packing import (
+ pack_awq_bitstream,
+ pack_scalar_zero,
+ requires_generic_bit_packing,
+ unpack_awq_bitstream,
+)
+
+# Bit-widths the AWQ packing layout can represent. AutoAWQ's classic layout
+# interleaves 8 values per int32 word, which only lines up at 4 bits; 5/6/7-bit
+# keep the same 8-value interleave but write it into the generic bit-stream
+# layout (exactly what humming's AWQWeightSchema expects).
+SUPPORTED_AWQ_BITS = (4, 5, 6, 7)
def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):
@@ -84,10 +96,16 @@ def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):
def dequantize_gemm(qweight, qzeros, scales, bits, group_size):
- # Unpack the qweight and qzeros tensors
- iweight, izeros = unpack_awq(qweight, qzeros, bits)
- # Reverse the order of the iweight and izeros tensors
- iweight, izeros = reverse_awq_order(iweight, izeros, bits)
+ if requires_generic_bit_packing(bits):
+ # 5/6/7-bit: values straddle int32 words, so unpack via the generic
+ # bit-stream helper (which also undoes the AWQ interleave).
+ iweight = unpack_awq_bitstream(qweight, bits, dim=1)
+ izeros = unpack_awq_bitstream(qzeros, bits, dim=1)
+ else:
+ # Unpack the qweight and qzeros tensors
+ iweight, izeros = unpack_awq(qweight, qzeros, bits)
+ # Reverse the order of the iweight and izeros tensors
+ iweight, izeros = reverse_awq_order(iweight, izeros, bits)
# overflow checks
iweight = torch.bitwise_and(iweight, (2**bits) - 1)
@@ -141,26 +159,32 @@ class WQLinear_GEMM(nn.Module):
def __init__(self, w_bit, group_size, in_features, out_features, bias, dev, training=False):
super().__init__()
- if w_bit not in [4]:
- raise NotImplementedError("Only 4-bit are supported for now.")
+ if w_bit not in SUPPORTED_AWQ_BITS:
+ raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_AWQ_BITS))}-bit are supported for now.")
self.in_features = in_features
self.out_features = out_features
self.w_bit = w_bit
self.group_size = group_size if group_size != -1 else in_features
self.training = training
+ self.use_generic_bit_packing = requires_generic_bit_packing(w_bit)
# quick sanity check (make sure alignment)
if self.in_features % self.group_size != 0:
raise ValueError(f"in_features ({self.in_features}) shape mismatch")
- if out_features % (32 // self.w_bit) != 0:
- raise ValueError(f"out_features ({out_features}) shape mismatch")
+ # 2/4/8-bit keep AutoAWQ's original ``32 // w_bit`` alignment rule; 5/6/7-bit
+ # pack a contiguous bit-stream over blocks of 32 values, so the output
+ # dimension must be a multiple of 32.
+ pack_align = 32 if self.use_generic_bit_packing else 32 // self.w_bit
+ if out_features % pack_align != 0:
+ raise ValueError(f"out_features ({out_features}) must be a multiple of {pack_align} for {w_bit}-bit AWQ")
+ packed_out_features = out_features * self.w_bit // 32
self.register_buffer(
"qweight",
torch.zeros(
- (in_features, out_features // (32 // self.w_bit)),
+ (in_features, packed_out_features),
dtype=torch.int32,
device=dev,
),
@@ -168,7 +192,7 @@ class WQLinear_GEMM(nn.Module):
self.register_buffer(
"qzeros",
torch.zeros(
- (in_features // self.group_size, out_features // (32 // self.w_bit)),
+ (in_features // self.group_size, packed_out_features),
dtype=torch.int32,
device=dev,
),
@@ -249,6 +273,22 @@ class WQLinear_GEMM(nn.Module):
intweight = intweight.to(dtype=torch.int32)
del repeat_scales
+ if awq_linear.use_generic_bit_packing:
+ # 5/6/7-bit: AWQ interleave (groups of 8) + generic bit-stream packing.
+ awq_linear.qweight = pack_awq_bitstream(intweight, awq_linear.w_bit, dim=1).to("cpu")
+ packed_cols = scales.shape[1] * awq_linear.w_bit // 32
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.to(dtype=torch.int32, device=device)
+ qzeros = pack_awq_bitstream(zeros, awq_linear.w_bit, dim=1)
+ else:
+ # A single shared zero point is interleave-invariant, so it can be
+ # tiled directly without reordering.
+ qzeros = pack_scalar_zero(
+ int(zeros), awq_linear.w_bit, packed_cols, (scales.shape[0], packed_cols), device=device
+ )
+ awq_linear.qzeros = qzeros.to("cpu")
+ return awq_linear
+
intweight = intweight.reshape(-1, intweight.shape[1] // pack_num, pack_num)
new_order_map = torch.tensor([0, 4, 1, 5, 2, 6, 3, 7], device=device) * awq_linear.w_bit
diff --git a/auto_round/export/formats/backends/auto_awq.py b/auto_round/export/formats/backends/auto_awq.py
index 1ab8c2831..00905cb5b 100644
--- a/auto_round/export/formats/backends/auto_awq.py
+++ b/auto_round/export/formats/backends/auto_awq.py
@@ -25,13 +25,17 @@ from auto_round.schemes import QuantizationScheme
@OutputFormat.register("auto_awq")
class AutoAWQFormat(OutputFormat):
- support_schemes = ["W4A16"]
+ support_schemes = ["W4A16", "W5A16", "W6A16", "W7A16"]
format_name = "auto_awq"
+ # See AutoGPTQFormat.EXTENDED_BITS -- 5/6/7-bit uses the generic bit-stream
+ # layout, which upstream AutoAWQ kernels cannot read.
+ EXTENDED_BITS = (5, 6, 7)
+
@classmethod
def check_scheme_args(cls, scheme: QuantizationScheme) -> bool:
error_logs = []
- if scheme.bits != 4:
+ if scheme.bits not in (4,) + cls.EXTENDED_BITS:
error_logs.append(f"bits={scheme.bits}")
if not re.search("int", scheme.data_type):
error_logs.append(f"data_type={scheme.data_type}")
@@ -49,14 +53,17 @@ class AutoAWQFormat(OutputFormat):
@staticmethod
def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):
"""Check whether a model is compatible with the AutoAWQ GEMM kernel."""
+ from auto_round.utils.bit_packing import requires_generic_bit_packing
from auto_round.utils.model import get_layer_names_in_block, get_module
- if bits != 4:
+ if bits not in (4, 5, 6, 7):
return False, "AutoAWQ GEMM kernel only supports 4 bits"
for _, module in model.named_modules():
if type(module) == transformers.pytorch_utils.Conv1D:
return False, "AutoAWQ GEMM kernel does not support conv1d"
+ # 5/6/7-bit packs a contiguous bit-stream over blocks of 32 values.
+ out_align = 32 if requires_generic_bit_packing(bits) else 32 // bits
layer_names = get_layer_names_in_block(model)
for layer_name in layer_names:
if (
@@ -69,8 +76,8 @@ class AutoAWQFormat(OutputFormat):
layer = get_module(model, layer_name)
if layer.in_features % group_size != 0:
return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"
- if layer.out_features % (32 // bits) != 0:
- return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"
+ if layer.out_features % out_align != 0:
+ return False, f"Layer {layer_name} out_features is not multiple of {out_align}"
return True, ""
@@ -80,8 +87,17 @@ class AutoAWQFormat(OutputFormat):
)
if not awq_supported:
logger.warning(f"The AutoAWQ format may not be supported due to {info}")
- if scheme.bits != 4:
- raise ValueError(f"auto_awq format support quantization scheme with W4A16 but got bits={scheme.bits}")
+ if scheme.bits in self.EXTENDED_BITS and not self.output_format.startswith("auto_round"):
+ raise ValueError(
+ f"{self.output_format} format does not support bits={scheme.bits}. "
+ f"{self.EXTENDED_BITS} bit packing is only defined for the `auto_round` format family, "
+ "please export to `auto_round:auto_awq`."
+ )
+ if scheme.bits not in (4,) + self.EXTENDED_BITS:
+ raise ValueError(
+ f"{self.format_name} format support quantization scheme with {','.join(self.support_schemes)} "
+ f"but got bits={scheme.bits}, please have a check."
+ )
if self.backend is None:
ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)
diff --git a/auto_round/export/formats/backends/auto_gptq.py b/auto_round/export/formats/backends/auto_gptq.py
index 4fcaaca61..03a14a33d 100644
--- a/auto_round/export/formats/backends/auto_gptq.py
+++ b/auto_round/export/formats/backends/auto_gptq.py
@@ -24,16 +24,36 @@ from auto_round.schemes import QuantizationScheme
@OutputFormat.register("auto_gptq", "gptqmodel")
class AutoGPTQFormat(OutputFormat):
- support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32", "W4A16_MIXED"]
+ support_schemes = [
+ "W4A16",
+ "W2A16",
+ "W3A16",
+ "W8A16",
+ "BF16",
+ "W2A16G64",
+ "W2A16G32",
+ "W4A16_MIXED",
+ ]
format_name = "auto_gptq"
+ # 5/6/7-bit weights straddle int32 word boundaries. AutoRound packs them with the
+ # generic bit-stream layout (see auto_round.utils.bit_packing), which upstream
+ # AutoGPTQ/GPTQModel kernels cannot read -- so restrict them to ``auto_round:*``.
+ EXTENDED_BITS = (5, 6, 7)
+
def check_and_reset_format(self, scheme: QuantizationScheme, ctx):
+ if scheme.bits in self.EXTENDED_BITS and not self.output_format.startswith("auto_round"):
+ raise ValueError(
+ f"{self.output_format} format does not support bits={scheme.bits}. "
+ f"{self.EXTENDED_BITS} bit packing is only defined for the `auto_round` format family, "
+ "please export to `auto_round` or `auto_round:auto_gptq`."
+ )
if not scheme.sym:
logger.warning(
"the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
" particularly for 2-bit quantization and smaller models."
" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
- "the AutoRound format(2/3/4/8 bits)."
+ "the AutoRound format(2/3/4/5/6/7/8 bits)."
)
if self.backend is None:
ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)
diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.py
index 4d570b26f..84134f6df 100644
--- a/auto_round/export/formats/backends/autoround.py
+++ b/auto_round/export/formats/backends/autoround.py
@@ -37,6 +37,9 @@ class AutoRoundFormat(OutputFormat):
"W4A16_MIXED",
"W2A16",
"W3A16",
+ "W5A16",
+ "W6A16",
+ "W7A16",
"W8A16",
"MXFP4",
"MXFP8",
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index dd6edd6fd..d45549a75 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -181,6 +181,31 @@ mxfp_nvfp_feature_checker = functools.partial(in_feature_checker_group_size)
ark_feature_checker = functools.partial(in_feature_checker_group_size)
+# humming repacks weights with ``padded_shape_n % 64 == 0`` / ``padded_shape_k % 32 == 0``.
+# humming can pad internally; requiring 32-multiples matches the checkpoint packing constraints.
+humming_feature_checker = functools.partial(
+ feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32
+)
+
+
+# humming is published only as a git repo (distribution name ``humming-kernels``),
+# so it cannot be expressed as a PEP 440 requirement string like the other backends.
+HUMMING_INSTALL_SPEC = "git+https://github.com/inclusionAI/humming.git"
+
+
+def humming_available_checker():
+ """Requirement hook: humming needs both the package and a CUDA device.
+
+ Returns ``(is_available, install_spec)``; the spec is fed to ``pip install`` in the
+ "better backend found" hint, so it must be the git URL rather than a version spec.
+ """
+ try:
+ from auto_round_extension.humming.qlinear_humming import is_humming_available
+
+ return is_humming_available(), HUMMING_INSTALL_SPEC
+ except Exception: # pragma: no cover - depends on the local environment
+ return False, HUMMING_INSTALL_SPEC
+
def fp8_static_scheme_checker(
in_feature: int,
@@ -452,7 +477,7 @@ BackendInfos["auto_round:torch"] = BackendInfo(
compute_dtype=["float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
- bits=[2, 3, 4, 8],
+ bits=[2, 3, 4, 5, 6, 7, 8],
priority=0,
checkers=[exllamav2_feature_checker],
alias=["auto_round", "torch"],
@@ -467,13 +492,62 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(
compute_dtype=["float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
- bits=[2, 3, 4, 8],
+ bits=[2, 3, 4, 5, 6, 7, 8],
priority=0,
checkers=[exllamav2_feature_checker],
alias=["torch", "torch_zp"],
requirements=["auto-round>=0.5.1"],
)
+# humming: JIT-compiled CUDA GEMM kernels covering every weight bit-width from
+# 2 to 8. This is the only fast path for 5/6/7-bit checkpoints, and it is also
+# faster than the Triton kernels for the classic 2/3/4/8-bit ones. Priority is
+# set above tritonv2 (2) but below marlin (6) so that existing 4-bit defaults
+# keep using marlin where it is available.
+BackendInfos["auto_round:humming"] = BackendInfo(
+ device=["cuda"],
+ sym=[True, False],
+ packing_format=GPTQ_FORMAT_NO_ZP,
+ compute_dtype=["float16", "bfloat16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ bits=[2, 3, 4, 5, 6, 7, 8],
+ priority=4,
+ checkers=[humming_feature_checker],
+ alias=["humming"],
+ requirements=[humming_available_checker],
+)
+
+BackendInfos["auto_round:humming_zp"] = BackendInfo(
+ device=["cuda"],
+ # asym `auto_round:auto_gptq` stores ``zp - 1``, which humming does not model;
+ # only the symmetric variant (where the zero point is implicit) is supported.
+ sym=[True],
+ packing_format=GPTQ_FORMAT,
+ compute_dtype=["float16", "bfloat16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ bits=[2, 3, 4, 5, 6, 7, 8],
+ priority=4,
+ checkers=[humming_feature_checker],
+ alias=["humming", "humming_zp"],
+ requirements=[humming_available_checker],
+)
+
+BackendInfos["auto_round:humming_awq"] = BackendInfo(
+ device=["cuda"],
+ sym=[True, False],
+ packing_format=AWQ_FORMAT,
+ compute_dtype=["float16", "bfloat16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ bits=[2, 3, 4, 5, 6, 7, 8],
+ priority=4,
+ checkers=[humming_feature_checker],
+ alias=["humming", "humming_awq"],
+ requirements=[humming_available_checker],
+)
+
BackendInfos["gptqmodel:marlin"] = BackendInfo(
device=["cuda"],
sym=[True],
@@ -853,6 +927,23 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):
if "auto_round:fake" in backend:
return ar_qmodules.FakeActQuantLinear
+ if "humming" in backend:
+ try:
+ import humming # noqa: F401 # pylint: disable=E0401
+ except Exception as e:
+ raise ImportError(
+ "Please install the humming kernels to use this backend, e.g.: "
+ "`pip install git+https://github.com/inclusionAI/humming.git`"
+ ) from e
+ import auto_round_extension.humming.qlinear_humming as humming_qlinear
+
+ if "awq" in backend:
+ return humming_qlinear.QuantLinearAWQ
+ elif "zp" in backend:
+ return humming_qlinear.QuantLinearGPTQ
+ else: # plain auto_round packing must be checked last
+ return humming_qlinear.QuantLinear
+
if "auto_round_kernel" in backend or "ark" in backend:
try:
import auto_round_kernel as ark # pylint: disable=E0611, E0401
@@ -1261,6 +1352,13 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="
# Filter requirements
missing_requirements = []
for req in requirements:
+ # Callable requirements (e.g. humming) report their own availability and
+ # return the spec to hand to `pip install`.
+ if callable(req):
+ available, install_spec = req()
+ if not available and install_spec:
+ missing_requirements.append(install_spec)
+ continue
try:
require_version(req)
except Exception:
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 491137040..7ebf1718b 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -523,7 +523,19 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,
bias=bias,
)
+ if "humming" in layer_backend:
+ return QuantLinear(
+ bits=config["bits"],
+ group_size=config["group_size"],
+ infeatures=in_features,
+ outfeatures=out_features,
+ bias=bias,
+ sym=config["sym"],
+ weight_dtype=layer.weight.dtype,
+ )
+
if "auto_round_kernel" in layer_backend:
+
return QuantLinear(
bits=config["bits"],
group_size=config["group_size"],
@@ -717,6 +729,7 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
need_autogptq_init = False
need_gptqmodel_init = False
need_ark_init = False
+ need_humming_init = False
used_gptq_exllamav2 = False
# Determine which backends require post-init
for backend in used_backends:
@@ -728,6 +741,8 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
need_gptqmodel_init = True
elif backend.startswith("auto_round_kernel"):
need_ark_init = True
+ elif "humming" in backend:
+ need_humming_init = True
# AutoGPTQ post-init
if need_autogptq_init:
@@ -770,6 +785,13 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
for layer in tqdm(layers, desc=message, total=len(layers), leave=True):
layer.post_init()
+ # humming post-init: repack the checkpoint layout into the kernel-native one
+ # and JIT-compile the selected GEMM.
+ if need_humming_init:
+ layers = [m for _, m in model.named_modules() if getattr(m, "QUANT_TYPE", "").startswith("humming")]
+ for layer in tqdm(layers, desc="repacking to humming format", total=len(layers), leave=True):
+ layer.post_init()
+
# ExLLaMAv2 kernels
if used_gptq_exllamav2:
_import_exllamav2_kernels()
diff --git a/auto_round/utils/bit_packing.py b/auto_round/utils/bit_packing.py
new file mode 100644
index 000000000..f444cc4d1
--- /dev/null
+++ b/auto_round/utils/bit_packing.py
@@ -0,0 +1,248 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic little-endian bit-stream packing for arbitrary weight bit-widths (1-8 bits).
+
+Rationale
+---------
+Both the classic GPTQ ``qweight`` layout and the AWQ ``qweight`` layout store
+sub-byte integers inside ``int32`` words. The historical AutoRound
+implementations special-cased ``bits in (2, 4, 8)`` (where ``32 % bits == 0``,
+so every value fits neatly inside a single word) and ``bits == 3`` (a bespoke
+hand-rolled routine). That is why 5/6/7-bit checkpoints could not be exported.
+
+In reality *all* of those layouts are the exact same thing: a contiguous
+little-endian bit-stream laid over blocks of **32 consecutive values**, which
+always yields exactly ``bits`` ``int32`` words per block::
+
+ value i (0 <= i < 32) occupies bits [i*bits, (i+1)*bits)
+ word = (i * bits) // 32
+ offset = (i * bits) % 32
+ # when offset + bits > 32 the value straddles ``word`` and ``word + 1``
+
+For ``bits in (2, 4, 8)`` no value ever straddles a word boundary, so this
+degenerates to the familiar ``32 // bits`` values-per-word layout. For
+``bits == 3`` it reproduces the legacy AutoRound/AutoGPTQ 3-bit routine
+bit-for-bit (10 values, a straddling value, 10 values, ... per 3 words). And
+for ``bits in (5, 6, 7)`` it "just works" -- no new concept is introduced.
+
+This is also precisely the layout consumed by the `humming
+<https://github.com/inclusionAI/humming>`_ GEMM kernels
+(``humming/include/humming/kernel/pack_weight.cuh::common_pack_weight``),
+which support any weight type below 8 bits. Sharing this single definition is
+what lets ``auto_round:auto_gptq`` / ``auto_round:auto_awq`` checkpoints carry
+5/6/7-bit weights and still be served by a fast kernel.
+
+Only the block size (32 values) is required to divide the packed dimension;
+there is no ``32 % bits == 0`` constraint.
+"""
+
+from typing import Optional
+
+import torch
+
+__all__ = [
+ "AWQ_PACK_ORDER",
+ "AWQ_REVERSE_ORDER",
+ "SUPPORTED_PACKING_BITS",
+ "awq_reverse_reorder",
+ "awq_reorder",
+ "pack_bitstream",
+ "unpack_bitstream",
+ "packed_dim_size",
+ "requires_generic_bit_packing",
+]
+
+# Weight bit-widths that ``pack_bitstream`` / ``unpack_bitstream`` handle.
+SUPPORTED_PACKING_BITS = (1, 2, 3, 4, 5, 6, 7, 8)
+
+# Number of source values covered by one packing block. ``PACK_BLOCK * bits``
+# is always a multiple of 32, so a block maps onto exactly ``bits`` int32 words.
+PACK_BLOCK = 32
+
+# AWQ shuffles values inside groups of 8 before writing them into the
+# bit-stream. ``AWQ_PACK_ORDER`` maps *logical* position -> *stored* slot and
+# ``AWQ_REVERSE_ORDER`` is its inverse (stored slot -> logical position).
+# For 4-bit these reproduce AutoAWQ's classic ``[0, 4, 1, 5, 2, 6, 3, 7]``
+# shift table exactly.
+AWQ_INTERLEAVE = 8
+AWQ_REVERSE_ORDER = (0, 4, 1, 5, 2, 6, 3, 7)
+AWQ_PACK_ORDER = (0, 2, 4, 6, 1, 3, 5, 7)
+
+_UINT32_MASK = 0xFFFFFFFF
+_INT32_SIGN = 1 << 31
+_UINT32_SPAN = 1 << 32
+
+
+def requires_generic_bit_packing(bits: int) -> bool:
+ """Whether ``bits`` needs the generic bit-stream path.
+
+ ``2/4/8`` (word-aligned) and ``3`` (legacy hand-rolled) keep their original
+ fast paths so that previously exported checkpoints stay byte-identical.
+ """
+ return int(bits) in (5, 6, 7)
+
+
+def packed_dim_size(num_values: int, bits: int) -> int:
+ """Number of ``int32`` words needed to store ``num_values`` values."""
+ if num_values % PACK_BLOCK != 0:
+ raise ValueError(
+ f"cannot pack {num_values} values at {bits} bits: the packed dimension must be a multiple of "
+ f"{PACK_BLOCK}. Pad the layer or leave it unquantized."
+ )
+ return num_values * bits // 32
+
+
+def _shift_tables(bits: int, device: torch.device):
+ """Per-value (word index, bit offset) tables for one 32-value block."""
+ idx = torch.arange(PACK_BLOCK, device=device, dtype=torch.int64) * bits
+ word = torch.div(idx, 32, rounding_mode="floor")
+ offset = idx % 32
+ return word, offset
+
+
+def _check_bits(bits: int) -> int:
+ bits = int(bits)
+ if bits not in SUPPORTED_PACKING_BITS:
+ raise ValueError(f"unsupported bits={bits} for bit-stream packing, expected one of {SUPPORTED_PACKING_BITS}")
+ return bits
+
+
+def pack_bitstream(values: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:
+ """Pack integer ``values`` into ``int32`` along ``dim`` using the bit-stream layout.
+
+ Args:
+ values: Integer tensor holding unsigned values in ``[0, 2**bits)``.
+ bits: Weight bit-width, see :data:`SUPPORTED_PACKING_BITS`.
+ dim: Dimension to pack. Its size must be a multiple of 32.
+
+ Returns:
+ ``int32`` tensor with ``dim`` shrunk to ``size * bits // 32``.
+ """
+ bits = _check_bits(bits)
+ values = values.movedim(dim, -1)
+ num_values = values.shape[-1]
+ num_words = packed_dim_size(num_values, bits)
+
+ lead_shape = values.shape[:-1]
+ # (..., n_blocks, 32) — one block produces exactly ``bits`` words.
+ values = values.reshape(*lead_shape, num_values // PACK_BLOCK, PACK_BLOCK).to(torch.int64)
+ values = values & ((1 << bits) - 1)
+
+ word, offset = _shift_tables(bits, values.device)
+ packed = torch.zeros(*values.shape[:-1], bits, dtype=torch.int64, device=values.device)
+
+ # Low part: the (possibly partial) chunk that lands in ``word``.
+ # Bit ranges never overlap, so accumulating with ``index_add_`` is
+ # equivalent to OR-ing but stays fully vectorized.
+ packed.index_add_(-1, word, values << offset)
+
+ straddles = (offset + bits) > 32
+ if bool(straddles.any()):
+ # High part: the remainder spills into the next word's low bits.
+ high = torch.where(straddles, values >> (32 - offset), torch.zeros_like(values))
+ packed.index_add_(-1, torch.clamp(word + 1, max=bits - 1), high)
+
+ packed = packed & _UINT32_MASK
+ # Reinterpret as signed int32 (torch has no uint32 storage).
+ packed = torch.where(packed >= _INT32_SIGN, packed - _UINT32_SPAN, packed).to(torch.int32)
+ packed = packed.reshape(*lead_shape, num_words)
+ return packed.movedim(-1, dim)
+
+
+def unpack_bitstream(packed: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:
+ """Inverse of :func:`pack_bitstream`.
+
+ Returns an ``int32`` tensor of unsigned values in ``[0, 2**bits)`` with
+ ``dim`` expanded back to ``num_words * 32 // bits``.
+ """
+ bits = _check_bits(bits)
+ packed = packed.movedim(dim, -1)
+ num_words = packed.shape[-1]
+ if num_words % bits != 0:
+ raise ValueError(f"packed dimension {num_words} is not a multiple of bits={bits}")
+
+ lead_shape = packed.shape[:-1]
+ packed = packed.reshape(*lead_shape, num_words // bits, bits).to(torch.int64) & _UINT32_MASK
+
+ word, offset = _shift_tables(bits, packed.device)
+ one = torch.ones((), dtype=torch.int64, device=packed.device)
+
+ low_bits = torch.clamp(torch.full_like(offset, 32) - offset, max=bits)
+ low = (packed.index_select(-1, word) >> offset) & (torch.bitwise_left_shift(one, low_bits) - 1)
+
+ high_bits = torch.full_like(low_bits, bits) - low_bits
+ if bool(torch.any(high_bits > 0)):
+ next_word = packed.index_select(-1, torch.clamp(word + 1, max=bits - 1))
+ high = torch.bitwise_left_shift(next_word & (torch.bitwise_left_shift(one, high_bits) - 1), low_bits)
+ low = low | torch.where(high_bits > 0, high, torch.zeros_like(high))
+
+ values = low.reshape(*lead_shape, num_words * 32 // bits).to(torch.int32)
+ return values.movedim(-1, dim)
+
+
+def _apply_awq_order(tensor: torch.Tensor, order, dim: int) -> torch.Tensor:
+ tensor = tensor.movedim(dim, -1)
+ size = tensor.shape[-1]
+ if size % AWQ_INTERLEAVE != 0:
+ raise ValueError(
+ f"AWQ interleaving requires the packed dimension ({size}) to be a multiple of {AWQ_INTERLEAVE}"
+ )
+ index = torch.as_tensor(order, dtype=torch.long, device=tensor.device)
+ tensor = tensor.reshape(*tensor.shape[:-1], size // AWQ_INTERLEAVE, AWQ_INTERLEAVE)
+ tensor = tensor.index_select(-1, index)
+ tensor = tensor.reshape(*tensor.shape[:-2], size)
+ return tensor.movedim(-1, dim)
+
+
+def awq_reorder(values: torch.Tensor, dim: int = -1) -> torch.Tensor:
+ """Shuffle logical values into AWQ storage order (apply before packing)."""
+ return _apply_awq_order(values, AWQ_PACK_ORDER, dim)
+
+
+def awq_reverse_reorder(values: torch.Tensor, dim: int = -1) -> torch.Tensor:
+ """Undo :func:`awq_reorder` (apply after unpacking)."""
+ return _apply_awq_order(values, AWQ_REVERSE_ORDER, dim)
+
+
+def pack_awq_bitstream(values: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:
+ """AWQ-flavoured packing: interleave groups of 8, then bit-stream pack."""
+ return pack_bitstream(awq_reorder(values, dim=dim), bits, dim=dim)
+
+
+def unpack_awq_bitstream(packed: torch.Tensor, bits: int, dim: int = -1) -> torch.Tensor:
+ """Inverse of :func:`pack_awq_bitstream`."""
+ return awq_reverse_reorder(unpack_bitstream(packed, bits, dim=dim), dim=dim)
+
+
+def pack_scalar_zero(value: int, bits: int, num_words: int, shape, device=None) -> torch.Tensor:
+ """Build a ``qzeros`` tensor where every group shares the same zero point.
+
+ ``value`` is broadcast to ``PACK_BLOCK`` entries and packed once, then the
+ resulting ``bits``-word pattern is tiled across ``num_words`` columns.
+ """
+ bits = _check_bits(bits)
+ block = torch.full((PACK_BLOCK,), int(value), dtype=torch.int64, device=device)
+ pattern = pack_bitstream(block, bits) # (bits,)
+ repeats = (num_words + bits - 1) // bits
+ row = pattern.repeat(repeats)[:num_words]
+ return row.reshape(*(1,) * (len(shape) - 1), num_words).expand(*shape).contiguous()
+
+
+def infer_packed_bits(num_values: int, num_words: int) -> Optional[int]:
+ """Best-effort recovery of ``bits`` from packed/unpacked dimension sizes."""
+ if num_values <= 0 or num_words <= 0:
+ return None
+ bits = num_words * 32 // num_values
+ return bits if bits in SUPPORTED_PACKING_BITS and num_values * bits == num_words * 32 else None
diff --git a/auto_round_extension/humming/__init__.py b/auto_round_extension/humming/__init__.py
new file mode 100644
index 000000000..1a48c8a45
--- /dev/null
+++ b/auto_round_extension/humming/__init__.py
@@ -0,0 +1,22 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round_extension.humming.qlinear_humming import (
+ QuantLinear,
+ QuantLinearAWQ,
+ QuantLinearGPTQ,
+ is_humming_available,
+)
+
+__all__ = ["QuantLinear", "QuantLinearAWQ", "QuantLinearGPTQ", "is_humming_available"]
diff --git a/auto_round_extension/humming/qlinear_humming.py b/auto_round_extension/humming/qlinear_humming.py
new file mode 100644
index 000000000..421920f8e
--- /dev/null
+++ b/auto_round_extension/humming/qlinear_humming.py
@@ -0,0 +1,234 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Inference backend backed by the `humming <https://github.com/inclusionAI/humming>`_ GEMM kernels.
+
+humming is a JIT-compiled quantized-GEMM library that supports **any** weight
+bit-width from 1 to 8 bits (SM75+), which makes it the natural kernel for
+AutoRound's 5/6/7-bit checkpoints -- the Triton/torch fallbacks either reject
+those bit-widths or dequantize them in Python.
+
+The checkpoint layouts AutoRound writes map one-to-one onto humming's schemas:
+
+=============================== ================================= ==================
+packing_format humming schema zero point
+=============================== ================================= ==================
+``auto_round`` / ``:gptqmodel`` ``GPTQWeightSchema`` stored as-is
+``auto_round:auto_gptq`` ``GPTQWeightSchema`` ``zp - 1`` (sym only)
+``auto_round:auto_awq`` ``AWQWeightSchema`` stored as-is
+=============================== ================================= ==================
+
+For symmetric quantization humming ignores ``qzeros`` entirely and implicitly
+dequantizes with ``(value - 2**(bits-1)) * scale`` (see
+``humming/utils/weight.py::dequantize_weight``), which is exactly AutoRound's
+full-range-symmetric convention -- so both the ``zp`` and ``zp - 1`` GPTQ
+variants land on the same (correct) numerics.
+"""
+
+import math
+from logging import getLogger
+
+import torch
+import torch.nn as nn
+
+logger = getLogger(__name__)
+
+# humming ships only as a git repo (distribution name ``humming-kernels``).
+HUMMING_INSTALL_SPEC = "git+https://github.com/inclusionAI/humming.git"
+
+# humming's weight transform asserts ``padded_shape_n % 64 == 0`` and
+# ``padded_shape_k % (2 * 256 // act_bits) == 0`` (32 for fp16/bf16 activations).
+PAD_N_TO_MULTIPLE = 64
+PAD_K_TO_MULTIPLE = 32
+
+SUPPORTED_BITS = (2, 3, 4, 5, 6, 7, 8)
+
+
+def is_humming_available() -> bool:
+ """Whether the humming kernels can be imported and used on this machine."""
+ try:
+ import humming # noqa: F401 # pylint: disable=E0401
+ except Exception: # pragma: no cover - depends on the local environment
+ return False
+ return torch.cuda.is_available()
+
+
+class QuantLinear(nn.Module):
+ """Weight-only quantized linear served by humming.
+
+ Buffers are registered with the *checkpoint* layout so that HuggingFace's
+ state-dict loader can fill them directly. ``post_init()`` then hands them to
+ humming, which repacks them into its kernel-native layout and releases the
+ originals.
+ """
+
+ QUANT_TYPE = "humming"
+ # packing_format string handed to humming's AutoRoundWeightSchema
+ PACKING_FORMAT = "auto_round"
+ # AWQ stores qweight transposed relative to GPTQ
+ AWQ_LAYOUT = False
+
+ def __init__(
+ self,
+ bits,
+ group_size,
+ infeatures,
+ outfeatures,
+ bias,
+ sym=True,
+ weight_dtype=torch.float16,
+ **kwargs,
+ ):
+ super().__init__()
+ if bits not in SUPPORTED_BITS:
+ raise NotImplementedError(f"humming backend supports {SUPPORTED_BITS} bits, got {bits}")
+ if infeatures % 32 != 0:
+ raise ValueError(f"humming requires in_features ({infeatures}) to be a multiple of 32")
+ if outfeatures % 32 != 0:
+ raise ValueError(f"humming requires out_features ({outfeatures}) to be a multiple of 32")
+
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.sym = bool(sym)
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+ self.compute_dtype = weight_dtype if weight_dtype in (torch.float16, torch.bfloat16) else torch.float16
+
+ num_groups = math.ceil(infeatures / self.group_size)
+ packed_out = outfeatures * self.bits // 32
+
+ if self.AWQ_LAYOUT:
+ qweight_shape = (infeatures, packed_out)
+ else:
+ qweight_shape = (infeatures * self.bits // 32, outfeatures)
+
+ self.register_buffer("qweight", torch.zeros(qweight_shape, dtype=torch.int32))
+ self.register_buffer("qzeros", torch.zeros((num_groups, packed_out), dtype=torch.int32))
+ self.register_buffer("scales", torch.zeros((num_groups, outfeatures), dtype=self.compute_dtype))
+ if bias:
+ self.register_buffer("bias", torch.zeros((outfeatures,), dtype=self.compute_dtype))
+ else:
+ self.bias = None
+
+ self.humming_layer = None
+
+ # ------------------------------------------------------------------ #
+ # setup
+ # ------------------------------------------------------------------ #
+ def _build_schema(self):
+ from humming.schema.autoround import AutoRoundWeightSchema # pylint: disable=E0401
+
+ return AutoRoundWeightSchema(
+ bits=self.bits,
+ group_size=self.group_size,
+ sym=self.sym,
+ data_type="int",
+ packing_format=self.PACKING_FORMAT,
+ act_bits=16,
+ )
+
+ def post_init(self):
+ """Hand the loaded checkpoint buffers to humming and repack them."""
+ if self.humming_layer is not None:
+ return
+
+ try:
+ from humming.layer import HummingLayer # pylint: disable=E0401
+ except Exception as e: # pragma: no cover - depends on the local environment
+ raise ImportError(
+ "The humming kernels are required for this backend "
+ "(and are the only real kernels for 5/6/7-bit weights). Install them with:\n"
+ f" pip install {HUMMING_INSTALL_SPEC}"
+ ) from e
+
+ device = self.qweight.device
+ if device.type != "cuda":
+ raise RuntimeError(f"humming kernels require a CUDA device, but the layer lives on {device}")
+
+ layer = HummingLayer(
+ shape_n=self.outfeatures,
+ shape_k=self.infeatures,
+ weight_config=self._build_schema(),
+ has_bias=self.bias is not None,
+ pad_n_to_multiple=PAD_N_TO_MULTIPLE,
+ pad_k_to_multiple=PAD_K_TO_MULTIPLE,
+ torch_dtype=self.compute_dtype,
+ )
+
+ tensors = {
+ "qweight": self.qweight,
+ "qzeros": self.qzeros,
+ "scales": self.scales.to(self.compute_dtype),
+ }
+ if self.bias is not None:
+ tensors["bias"] = self.bias.to(self.compute_dtype)
+ layer.load_from_tensors(tensors)
+ layer = layer.to(device)
+ # JIT-compiles / selects the kernel and rewrites the weights in place.
+ layer.transform()
+
+ # Release the checkpoint-layout buffers; humming owns the weights now.
+ for name in ("qweight", "qzeros", "scales", "bias"):
+ if name in self._buffers:
+ self._buffers.pop(name)
+ self.bias = None
+ self.humming_layer = layer
+
+ def pack(self, linear, scales, zeros, g_idx=None, device=None):
+ raise NotImplementedError(
+ "the humming backend is inference-only; export with the `auto_round`, "
+ "`auto_round:auto_gptq` or `auto_round:auto_awq` format instead"
+ )
+
+ # ------------------------------------------------------------------ #
+ # forward
+ # ------------------------------------------------------------------ #
+ def forward(self, x):
+ if self.humming_layer is None:
+ self.post_init()
+
+ out_shape = x.shape[:-1] + (self.outfeatures,)
+ x_dtype = x.dtype
+ x = x.reshape(-1, x.shape[-1]).to(self.compute_dtype)
+ out = self.humming_layer(x)
+ # humming may pad the N dimension up to a multiple of 64.
+ if out.shape[-1] != self.outfeatures:
+ out = out[..., : self.outfeatures]
+ return out.reshape(out_shape).to(x_dtype)
+
+
+class QuantLinearGPTQ(QuantLinear):
+ """``auto_round:auto_gptq`` packing (``qzeros`` stores ``zp - 1``)."""
+
+ QUANT_TYPE = "humming_gptq"
+ PACKING_FORMAT = "auto_round:auto_gptq"
+ AWQ_LAYOUT = False
+
+
+class QuantLinearAWQ(QuantLinear):
+ """``auto_round:auto_awq`` packing (interleaved, transposed qweight)."""
+
+ QUANT_TYPE = "humming_awq"
+ PACKING_FORMAT = "auto_round:auto_awq"
+ AWQ_LAYOUT = True
+
+
+__all__ = [
+ "QuantLinear",
+ "QuantLinearAWQ",
+ "QuantLinearGPTQ",
+ "SUPPORTED_BITS",
+ "is_humming_available",
+]
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index 0c6bd0d58..466057700 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -21,9 +21,18 @@ import torch.nn as nn
import transformers
from auto_round.utils import get_packing_device
+from auto_round.utils.bit_packing import (
+ pack_bitstream,
+ pack_scalar_zero,
+ requires_generic_bit_packing,
+ unpack_bitstream,
+)
logger = getLogger(__name__)
+# Bit-widths this QuantLinear can pack/unpack. 2/4/8 are word-aligned, 3 uses the
+# legacy hand-rolled routine, and 5/6/7 go through the generic bit-stream helpers.
+SUPPORTED_BITS = (2, 3, 4, 5, 6, 7, 8)
_DEVICE_WF_3BITS_TENSORS = {}
# Constants for FP4 values (E2M1 format)
@@ -44,7 +53,62 @@ def get_wf_3bits_tensor(device):
return _DEVICE_WF_3BITS_TENSORS[device_str]
-class QuantLinear(nn.Module):
+class GenericBitPackingMixin:
+ """Shared 5/6/7-bit packing for the GPTQ-style QuantLinear variants.
+
+ Both ``qlinear_torch`` (raw zero point) and ``qlinear_torch_zp`` (GPTQ's
+ ``zp - 1`` convention) produce the same bit layout, differing only by a
+ constant offset applied to the zero points.
+ """
+
+ def _to_intweight(self, linear, scales, zeros, device):
+ """Dequantization-domain -> integer grid, shared by the packing routines."""
+ W = linear.weight.data.to(device).clone()
+ if type(linear) == nn.Conv2d:
+ W = W.flatten(1)
+ if type(linear) == transformers.pytorch_utils.Conv1D:
+ W = W.t()
+
+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
+ if isinstance(zeros, torch.Tensor):
+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])
+ else:
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + zeros)
+ return intweight.to(torch.int32)
+
+ def pack_generic_bits(self, linear, scales, zeros, g_idx=None, device=None, zp_offset=0):
+ """Pack 5/6/7-bit weights using the generic little-endian bit-stream layout.
+
+ The resulting ``qweight``/``qzeros`` follow exactly the same convention as
+ the 2/3/4/8-bit paths (values laid out contiguously over blocks of 32
+ along the packed dimension), so downstream kernels -- including humming --
+ can consume every bit-width with one code path.
+
+ ``zp_offset`` is ``-1`` for the GPTQ-style "zero point minus one" variant.
+ """
+ device = get_packing_device(device)
+ scales_t = scales.t().contiguous()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ self.scales = scales_t.clone().half()
+
+ intweight = self._to_intweight(linear, scales, zeros, device)
+ # (out, in) -> pack along ``in`` -> transpose to the (in*bits/32, out) layout
+ self.qweight = pack_bitstream(intweight, self.bits, dim=1).t().contiguous().to("cpu")
+ del intweight
+
+ num_zero_words = scales_t.shape[1] * self.bits // 32
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.t().contiguous().to(device).to(torch.int32) + zp_offset
+ self.qzeros = pack_bitstream(zeros, self.bits, dim=1).to("cpu")
+ else:
+ self.qzeros = pack_scalar_zero(
+ int(zeros) + zp_offset, self.bits, num_zero_words, (scales_t.shape[0], num_zero_words)
+ ).to("cpu")
+
+
+class QuantLinear(GenericBitPackingMixin, nn.Module):
"""
Torch quantized linear layer.
"""
@@ -53,8 +117,8 @@ class QuantLinear(nn.Module):
def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):
super().__init__()
- if bits not in [2, 3, 4, 8]:
- raise NotImplementedError("Only 2,3,4,8 bits are supported.")
+ if bits not in SUPPORTED_BITS:
+ raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")
self.infeatures = infeatures
self.outfeatures = outfeatures
self.bits = bits
@@ -95,13 +159,18 @@ class QuantLinear(nn.Module):
self.trainable = trainable
# is performed by unpacking the weights and using torch.matmul
- if self.bits in [2, 4, 8]:
+ self.use_generic_bit_packing = requires_generic_bit_packing(self.bits)
+ if self.use_generic_bit_packing:
+ # 5/6/7-bit values straddle int32 word boundaries, so a single shift
+ # table cannot describe them; unpacking goes through the generic helper.
+ self.wf = None
+ elif self.bits in [2, 4, 8]:
self.wf = torch.tensor(
list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device
).unsqueeze(0)
elif self.bits == 3:
self.wf = get_wf_3bits_tensor(device=self.qweight.device)
- self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8
+ self.dequant_dtype = torch.int16 if self.bits > 7 else torch.int8
def post_init(self):
pass
@@ -291,15 +360,21 @@ class QuantLinear(nn.Module):
return self.pack_248_bits(linear, scales, zeros, g_idx, device)
elif self.bits in [3]:
return self.pack_3bits(linear, scales, zeros, g_idx, device)
+ elif self.use_generic_bit_packing:
+ return self.pack_generic_bits(linear, scales, zeros, g_idx, device)
else:
- raise ValueError("Only 2,3,4,8 bits are supported.")
+ raise ValueError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")
def forward(self, x):
out_shape = x.shape[:-1] + (self.outfeatures,)
x = x.reshape(-1, x.shape[-1])
x_dtype = x.dtype
- if self.bits in [2, 4, 8]:
+ if self.use_generic_bit_packing:
+ # Generic bit-stream layout: unpack straight into 2-D tensors.
+ zeros = unpack_bitstream(self.qzeros, self.bits, dim=1).reshape(self.scales.shape)
+ weight = unpack_bitstream(self.qweight, self.bits, dim=0)
+ elif self.bits in [2, 4, 8]:
if self.wf.device != self.qzeros.device:
self.wf = torch.tensor(
list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device
@@ -341,7 +416,8 @@ class QuantLinear(nn.Module):
weight = weight & 0x7
weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)
- weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
+ if weight.dim() == 3:
+ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
if hasattr(self, "g_idx"):
num_itr = self.g_idx.shape[0] // x.shape[-1]
num_dim = self.g_idx.shape[0] // num_itr
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index 5da317b71..d6fef7beb 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -21,12 +21,17 @@ import torch.nn as nn
import transformers
from auto_round.utils import get_packing_device
-from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensor
+from auto_round.utils.bit_packing import requires_generic_bit_packing, unpack_bitstream
+from auto_round_extension.torch.qlinear_torch import (
+ SUPPORTED_BITS,
+ GenericBitPackingMixin,
+ get_wf_3bits_tensor,
+)
logger = getLogger(__name__)
-class QuantLinear(nn.Module):
+class QuantLinear(GenericBitPackingMixin, nn.Module):
"""
Torch quantized linear layer.
"""
@@ -35,8 +40,8 @@ class QuantLinear(nn.Module):
def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):
super().__init__()
- if bits not in [2, 3, 4, 8]:
- raise NotImplementedError("Only 2,3,4,8 bits are supported.")
+ if bits not in SUPPORTED_BITS:
+ raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")
self.infeatures = infeatures
self.outfeatures = outfeatures
self.bits = bits
@@ -77,14 +82,18 @@ class QuantLinear(nn.Module):
self.trainable = trainable
# is performed by unpacking the weights and using torch.matmul
- if self.bits in [2, 4, 8]:
+ self.use_generic_bit_packing = requires_generic_bit_packing(self.bits)
+ if self.use_generic_bit_packing:
+ # 5/6/7-bit values straddle int32 word boundaries; use the generic helper.
+ self.wf = None
+ elif self.bits in [2, 4, 8]:
self.wf = torch.tensor(
list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device
).unsqueeze(0)
else: ## bits == 3
self.wf = get_wf_3bits_tensor(device=self.qweight.device)
- self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8
+ self.dequant_dtype = torch.int16 if self.bits > 7 else torch.int8
def post_init(self):
pass
@@ -288,15 +297,22 @@ class QuantLinear(nn.Module):
return self.pack_248_bits(linear, scales, zeros, g_idx, device)
elif self.bits in [3]:
return self.pack_3bits(linear, scales, zeros, g_idx, device)
+ elif self.use_generic_bit_packing:
+ # GPTQ stores ``zp - 1``; reuse the shared generic packer with that offset.
+ return self.pack_generic_bits(linear, scales, zeros, g_idx, device, zp_offset=-1)
else:
- raise ValueError("Only 2,3,4,8 bits are supported.")
+ raise ValueError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")
def forward(self, x):
out_shape = x.shape[:-1] + (self.outfeatures,)
x = x.reshape(-1, x.shape[-1])
x_dtype = x.dtype
- if self.bits in [2, 4, 8]:
+ if self.use_generic_bit_packing:
+ # Generic bit-stream layout: unpack straight into 2-D tensors.
+ zeros = unpack_bitstream(self.qzeros, self.bits, dim=1).reshape(self.scales.shape)
+ weight = unpack_bitstream(self.qweight, self.bits, dim=0)
+ elif self.bits in [2, 4, 8]:
if self.wf.device != self.qzeros.device:
self.wf = torch.tensor(
list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device
@@ -338,7 +354,8 @@ class QuantLinear(nn.Module):
weight = weight & 0x7
weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)
zeros += 1
- weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
+ if weight.dim() == 3:
+ weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
if hasattr(self, "g_idx"):
num_itr = self.g_idx.shape[0] // x.shape[-1]
num_dim = self.g_idx.shape[0] // num_itr
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 03a5614af..682f2bdb2 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -123,9 +123,9 @@ AutoRound supports several Schemes:
- **W4A16**(bits:4,group_size:128,sym:True,act_bits:16)
- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)
-- **W7A16**(bits:7,group_size:128,sym:True,act_bits:16) - `llm_compressor` format
-- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats
-- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats
+- **W7A16**(bits:7,group_size:128,sym:True,act_bits:16)
+- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16)
+- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16)
- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)
- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)
- **Group-size variants** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor` (W2-W8); `auto_round` / `auto_gptq` (W2 only)
@@ -146,7 +146,17 @@ Besides, you could modify the `group_size`, `bits`, `sym` and many other configs
You can use command `auto_round list format` to show all supported formats with support scheme.
**AutoRound Format**: This format is well-suited for CPU, Intel GPU, CUDA and HPU devices, 2 bits, as well as mixed-precision
-inference. **[2,3,4,8] bits are supported**. Please set `--format auto_round`
+inference. **[2,3,4,5,6,7,8] bits are supported**. Please set `--format auto_round`
+
+> **About 5/6/7 bits**: these widths do not divide 32 evenly, so each value is stored in a contiguous
+> little-endian bit-stream laid over blocks of 32 values (`in_features * bits / 32` int32 words).
+> This is a strict generalization of the existing layouts — 2/4/8-bit and 3-bit checkpoints keep
+> byte-identical `qweight`/`qzeros`. 5/6/7 bits work with `auto_round`, `auto_round:auto_gptq` and
+> `auto_round:auto_awq`, and can be freely combined in mixed-bit recipes. Upstream AutoGPTQ/AutoAWQ
+> kernels cannot read them, so the plain `auto_gptq` / `auto_awq` formats still reject 5/6/7 bits.
+> Both `in_features` and `out_features` must be multiples of 32. For inference, install the
+> [humming](https://github.com/inclusionAI/humming) kernels (`pip install git+https://github.com/inclusionAI/humming.git`);
+> otherwise AutoRound falls back to the slow `torch` dequantize-and-matmul backend.
**GGUF** Format: Experimental feature. This format is well-suited for CPU devices and is widely adopted by the
community. `q*_k`,`q*_0`,`q*_1` are supported. Please set `--format gguf:q4_k_m`, `--format gguf:q2_k_s`, etc
@@ -158,6 +168,7 @@ models. Besides, recently 3 bits may have some accuracy issues in Transformers.
**AutoAWQ Format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widely
adopted within the community, **only 4-bits quantization is supported**. Please set `--format auto_awq`
+(the AutoRound-flavoured `auto_round:auto_awq` additionally supports 5/6/7 bits).
**LLM-Compressor Format**: **NVFP4, MXFP4(kernel in WIP), MXFP8 are supported**. Please set `--format llm_compressor`
@@ -171,10 +182,12 @@ adopted within the community, **only 4-bits quantization is supported**. Please
| Format | Supported Schemes |
|:-----------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16`, `MXINT4` |
+| **auto_round** | W4A16, W2A16, W3A16, W5A16, W6A16, W7A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16`, `MXINT4` |
| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK, W2A16, W3A16, W4A16, W5A16, W6A16, W7A16, W8A16 and G64/G32 variants of W2-W8 |
| **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 |
| **mlx** / **auto_round:mlx** | W2A16, W3A16, W4A16, W5A16, W6A16, W8A16, BF16, mixed-bit / mixed-group_size (Apple Silicon only) |
+| **auto_round:auto_awq** | W4A16, W5A16, W6A16, W7A16, BF16 |
+| **auto_round:auto_gptq** | W4A16, W2A16, W3A16, W5A16, W6A16, W7A16, W8A16, W2A16G64, W2A16G32, BF16 |
| **auto_awq** | W4A16, BF16 |
| **auto_gptq** | W4A16, W2A16, W3A16, W8A16,W2A16G64, W2A16G32, BF16 |
| **fp8** | FP8_BLOCK |
@@ -489,7 +502,7 @@ auto_round \
#### API Usage
~~~
avg_bits= 3.0
-scheme = AutoScheme(avg_bits=avg_bits, options=("W2A16G64“, "W4A16","W8A16"))
+scheme = AutoScheme(avg_bits=avg_bits, options=("W2A16G64", "W4A16", "W8A16"))
ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
ar.quantize_and_save()
~~~
@@ -1019,7 +1032,12 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
### CUDA
-Supports 2, 3, 4, and 8 bits. We recommend using GPTQModel for 4 and 8 bits inference.
+Supports 2, 3, 4, 5, 6, 7 and 8 bits. We recommend using GPTQModel for 4 and 8 bits inference, and
+[humming](https://github.com/inclusionAI/humming) for 5/6/7 bits (it is also faster than Triton for the other widths).
+
+```bash
+pip install git+https://github.com/inclusionAI/humming.git
+```
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -1088,8 +1106,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| gptqmodel:awq_marlin | cuda | 4,8 | FP16 | 5 | awq | gptqmodel |
| gptqmodel:awq_gemm | cuda | 4 | FP16 | 3 | awq | gptqmodel |
| gptqmodel:awq_torch | cuda/cpu | 4 | FP16 | 2 | awq | gptqmodel |
+| humming | cuda | 2,3,4,5,6,7,8 | BF16/FP16 | 4 | gptq/gptq_zp+-1 | humming-kernels |
+| humming | cuda | 2,3,4,5,6,7,8 | BF16/FP16 | 4 | awq | humming-kernels |
| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
-| torch | xpu/cpu/cuda | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
+| torch | xpu/cpu/cuda | 2,3,4,5,6,7,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
### Convert GPTQ/AWQ to AutoRound
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 7d0f3d5de..9a0563822 100755
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -117,9 +117,9 @@ pip install auto-round
AutoRound 支持多种量化配置:
- **W4A16**(bits:4, group_size:128, sym:True, act_bits:16) # 4位权重,分组大小为128,对称量化,16位激活,
- **W8A16**(bits:8, group_size:128, sym:True, act_bits:16)
-- **W7A16**(bits:7, group_size:128, sym:True, act_bits:16) - 仅 `llm_compressor` 格式
-- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式
-- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式
+- **W7A16**(bits:7, group_size:128, sym:True, act_bits:16)
+- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16)
+- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16)
- **W3A16**(bits:3, group_size:128, sym:True, act_bits:16)
- **W2A16**(bits:2, group_size:128, sym:True, act_bits:16)
- **分组大小变体** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor`(W2-W8);`auto_round` / `auto_gptq`(仅 W2)
@@ -141,13 +141,21 @@ AutoRound 支持多种量化配置:
执行 `auto_round list format` 可查看所有支持的导出格式及其对应的量化方案。
-**AutoRound 原生格式**:适用于 CPU、英特尔 GPU、CUDA、HPU 等设备,支持2位宽及混合精度推理,**兼容 [2、3、4、8] bits**。使用时需设置 `--format auto_round`。
+**AutoRound 原生格式**:适用于 CPU、英特尔 GPU、CUDA、HPU 等设备,支持2位宽及混合精度推理,**兼容 [2、3、4、5、6、7、8] bits**。使用时需设置 `--format auto_round`。
+
+> **关于 5/6/7 bits**:这几种位宽无法整除 32,因此每个值被存放在以 32 个元素为一组、按小端连续排布的比特流中
+> (共 `in_features * bits / 32` 个 int32)。这是对现有排布的严格推广——2/4/8 bit 与 3 bit 的
+> `qweight`/`qzeros` 与之前完全逐字节一致。5/6/7 bits 可用于 `auto_round`、`auto_round:auto_gptq`
+> 和 `auto_round:auto_awq`,也可以自由用于混合 bit 配方。上游 AutoGPTQ/AutoAWQ 的 kernel 无法读取该排布,
+> 因此原生的 `auto_gptq` / `auto_awq` 格式仍然拒绝 5/6/7 bits。`in_features` 与 `out_features` 都必须是 32 的倍数。
+> 推理时请安装 [humming](https://github.com/inclusionAI/humming) kernel
+> (`pip install git+https://github.com/inclusionAI/humming.git`),否则会回退到较慢的 `torch` 反量化后再矩乘的后端。
**GGUF 格式**:实验性功能,适用于 CPU 设备,是社区主流格式之一,支持 `q*_k`、`q*_0`、`q*_1` 系列的量化。需设置 `--format gguf:q4_k_m`、`--format gguf:q2_k_s`等具体格式。
**AutoGPTQ 格式**:适用于 CUDA 设备的对称量化,在社区中广泛应用,**兼容 [2、3、4、8] bits **(但其**非对称推理核存在问题**,可能导致模型的精度大幅下降,尤其是在 2-bit 量化和小模型的场景;近期 Transformers 框架中 3-bits 量化也存在类似问题)。配置时需设置 `--format auto_gptq`。
-**AutoAWQ 格式**:适用于 CUDA 设备的 4 位非对称量化,在社区中也广泛应用。**仅支持 4-bit 量化**。需设置 `--format auto_awq`。
+**AutoAWQ 格式**:适用于 CUDA 设备的 4 位非对称量化,在社区中也广泛应用。**仅支持 4-bit 量化**。需设置 `--format auto_awq`(AutoRound 风格的 `auto_round:auto_awq` 额外支持 5/6/7 bits)。
**LLM-Compressor 格式**:**支持 NVFP4、MXFP4(kernel 开发中)、MXFP8** 等。需设置 `--format llm_compressor`。
@@ -163,9 +171,11 @@ AutoRound 支持多种量化配置:
| 格式 | 支持的量化方案 |
|:-------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` |
+| **auto_round** | W4A16、W2A16、W3A16、W5A16、W6A16、W7A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` |
| **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 |
| **llm_compressor** | NVFP4、W2A16、W3A16、W4A16、W5A16、W6A16、W7A16、W8A16 及 W2-W8 的 G64/G32 变体、FP8_BLOCK、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` |
+| **auto_round:auto_awq** | W4A16、W5A16、W6A16、W7A16、BF16 |
+| **auto_round:auto_gptq** | W4A16、W2A16、W3A16、W5A16、W6A16、W7A16、W8A16、W2A16G64、W2A16G32、BF16 |
| **auto_awq** | W4A16、BF16 |
| **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 |
| **mlx** / **auto_round:mlx** (实验性功能) | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) |
@@ -977,7 +987,14 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
```
### CUDA
-支持 2、3、4、8 bits 量化模型,**4-bits/8-bits 推理推荐使用 GPTQModel 后端**,使用示例:
+支持 2、3、4、5、6、7、8 bits 量化模型,**4-bits/8-bits 推理推荐使用 GPTQModel 后端**;**5/6/7 bits 推荐使用
+[humming](https://github.com/inclusionAI/humming) 后端**(它在其他位宽上也比 Triton 更快):
+
+```bash
+pip install git+https://github.com/inclusionAI/humming.git
+```
+
+使用示例:
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -1049,8 +1066,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| gptqmodel:awq_marlin | cuda | 4、8 | FP16 | 5 | awq | gptqmodel |
| gptqmodel:awq_gemm | cuda | 4 | FP16 | 3 | awq | gptqmodel |
| gptqmodel:awq_torch | cuda/cpu | 4 | FP16 | 2 | awq | gptqmodel |
+| humming | cuda | 2、3、4、5、6、7、8 | BF16/FP16 | 4 | gptq/gptq_zp+-1 | humming-kernels |
+| humming | cuda | 2、3、4、5、6、7、8 | BF16/FP16 | 4 | awq | humming-kernels |
| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
-| torch | xpu/cpu/cuda | 2、3、4、8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
+| torch | xpu/cpu/cuda | 2、3、4、5、6、7、8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
### 将 GPTQ 或 AWQ 模型转换为 AutoRound 格式
为了提升兼容性(尤其是英特尔设备),大部分 GPTQ/AWQ 量化模型均可转换为 AutoRound 格式。**注意**:若模型再次存储,其量化配置可能会发生变更, 由 gptq/awq 量化变化成 auto-round 量化。
diff --git a/test/unit/test_cpu/export/test_export.py b/test/unit/test_cpu/export/test_export.py
index 0822782b9..e2310fab0 100644
--- a/test/unit/test_cpu/export/test_export.py
+++ b/test/unit/test_cpu/export/test_export.py
@@ -556,7 +556,10 @@ class TestAutoRound:
sym=True,
)
ar.post_init()
- with pytest.raises(ValueError, match="auto_awq format support quantization scheme with W4A16 but got bits=2"):
+ with pytest.raises(
+ ValueError,
+ match="auto_awq format support quantization scheme with W4A16,W5A16,W6A16,W7A16 but got bits=2",
+ ):
resolve_formats(
ResolvedScheme.from_scheme(ar.scheme_context),
format="auto_round:auto_awq",