Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

improved algorithm for int2 (#748)

Wenhua Cheng64e76116e826
diff --git a/README.md b/README.md
index 0e5c3539f..cfd900665 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,10 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
## 🆕 What's New
+[202508] AutoRound now provides experimental support for an improved INT2 algorithm via `--enable_alg_ext`. See this [documentation](./docs/alg_202508.md)
+ for some accuracy results.
+
+
[2025/07] AutoRound now offers experimental support for **GGUF** format, and recommends using optimized RTN mode (--iters 0) for
all bits other than 3 bits. Example
models: [Intel/Qwen3-235B-A22B-q2ks-mixed-ar](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-ar)
diff --git a/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so b/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so
new file mode 100644
index 000000000..2798b41d8
Binary files /dev/null and b/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so differ
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index c4915f779..d53ce64ad 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -177,6 +177,7 @@ class AutoRound(object):
enable_torch_compile: bool = False,
device_map: Union[str, dict] = None,
disable_opt_rtn: bool = False,
+ enable_alg_ext: bool = False,
**kwargs,
):
## to ensure backward compatibility, move infrequently used arguments to kwargs arguments.
@@ -225,6 +226,7 @@ class AutoRound(object):
)
model = _handle_moe_model(model)
# self.model_orig_dtype = model.dtype
+ self.enable_alg_ext = enable_alg_ext
self.device = detect_device(device) ##must place after llm_load_model, because this one will convert auto
## important tuning hype-parameters
@@ -2566,10 +2568,35 @@ class AutoRound(object):
elif isinstance(input_others[key], list):
for i in range(len(input_others[key])):
to_dtype(input_others[key][i], tmp_dtype)
- if self.enable_torch_compile:
- quant_block = compile_func(self.quantize_block, device)
+
+ if (
+ self.act_bits > 8
+ and self.sym
+ and self.enable_alg_ext
+ and self.super_group_size is None
+ and self.data_type.startswith("int")
+ ):
+ try:
+ from auto_round.alg_ext import quantize_block_ext
+
+ AutoRound.quantize_block_ext = quantize_block_ext
+ quantize_block = self.quantize_block_ext ##must use self.quantize_block_ext
+ if self.bits > 2:
+ logger.warning(
+ "algorithm extension has only undergone limited validation on INT2; use with caution."
+ )
+ else:
+ logger.info("using algorithm extension for quantization.")
+ except (ImportError, ModuleNotFoundError):
+ quantize_block = self.quantize_block
+ if self.enable_torch_compile:
+ quantize_block = compile_func(quantize_block, device)
+ else:
+ quantize_block = quantize_block
else:
- quant_block = self.quantize_block
+ quantize_block = self.quantize_block
+ if self.enable_torch_compile:
+ quantize_block = compile_func(quantize_block, device)
if pbar is None:
pbar = tqdm(range(0, len(block_names), nblocks))
@@ -2590,7 +2617,7 @@ class AutoRound(object):
if not self.model.device.type == "meta" or self.low_cpu_mem_usage:
m = m.to(device)
- q_input, input_ids = quant_block(
+ q_input, input_ids = quantize_block(
m,
input_ids,
input_others,
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 08b693c26..76177e5bb 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -198,6 +198,8 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")
+ self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")
+
self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")
self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")
@@ -565,6 +567,7 @@ def tune(args):
super_bits=args.super_bits,
disable_opt_rtn=args.disable_opt_rtn,
disable_deterministic_algorithms=args.disable_deterministic_algorithms,
+ enable_alg_ext=args.enable_alg_ext,
**mllm_kwargs,
)
@@ -713,6 +716,8 @@ def tune(args):
args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype
)
st = time.time()
+ if "llama" in args.model.lower():
+ model_args += ",add_bos_token=True"
res = simple_evaluate(
model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs
)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 4f47a3aa4..b9462a500 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -15,11 +15,11 @@
import collections.abc
import copy
import gc
+import importlib
import json
import logging
import os
import re
-import subprocess
import sys
from collections import UserDict
from enum import Enum
@@ -27,7 +27,6 @@ from functools import lru_cache
from typing import Any, Tuple, Union
import cpuinfo
-import psutil
import torch
import transformers
from packaging import version
@@ -129,10 +128,6 @@ fh = logging.StreamHandler()
fh.setFormatter(AutoRoundFormatter())
logger.addHandler(fh)
-import importlib
-
-import transformers
-
class LazyImport(object):
"""Lazy import python module till use."""
diff --git a/docs/alg_202508.md b/docs/alg_202508.md
new file mode 100644
index 000000000..3fb74ace8
--- /dev/null
+++ b/docs/alg_202508.md
@@ -0,0 +1,14 @@
+We use **lm-eval** for evaluation. For LLaMA, we enabled `add_bos_token` and
+`removed @use_kernel_forward_from_hub("RMSNorm")`
+in [modeling_llama.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L52C1-L52C40)
+to stabilize accuracy during evaluation. All other settings follow the default configurations of AutoRound and lm-eval.
+
+| Qwen3-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande |
+|----------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------|
+| origin | 0.4373 | 0.4019 | 0.4437 | 0.4215 | 0.4826 | 0.5474 | 0.263 | 0.3072 | 0.6314 |
+| new | 0.4787 | 0.4275 | 0.4516 | 0.5944 | 0.5181 | 0.5773 | 0.2807 | 0.3305 | 0.6496 |
+
+| Llama3.1-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande |
+|-------------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------|
+| origin | 0.382 | 0.3635 | 0.4562 | 0.1622 | 0.5069 | 0.4411 | 0.1661 | 0.3207 | 0.6393 |
+| new | 0.4166 | 0.3712 | 0.4729 | 0.2039 | 0.5946 | 0.4981 | 0.2163 | 0.3011 | 0.6748 |
\ No newline at end of file