PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
improved algorithm for int2 (#748)
Wenhua Cheng64e76116e826
diff --git a/README.md b/README.mdindex 0e5c3539f..cfd900665 100644--- a/README.md+++ b/README.md@@ -28,6 +28,10 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).## 🆕 What's New+[202508] AutoRound now provides experimental support for an improved INT2 algorithm via `--enable_alg_ext`. See this [documentation](./docs/alg_202508.md)+ for some accuracy results.++[2025/07] AutoRound now offers experimental support for **GGUF** format, and recommends using optimized RTN mode (--iters 0) forall bits other than 3 bits. Examplemodels: [Intel/Qwen3-235B-A22B-q2ks-mixed-ar](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-ar)diff --git a/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so b/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.sonew file mode 100644index 000000000..2798b41d8Binary files /dev/null and b/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so differdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c4915f779..d53ce64ad 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -177,6 +177,7 @@ class AutoRound(object):enable_torch_compile: bool = False,device_map: Union[str, dict] = None,disable_opt_rtn: bool = False,+ enable_alg_ext: bool = False,**kwargs,):## to ensure backward compatibility, move infrequently used arguments to kwargs arguments.@@ -225,6 +226,7 @@ class AutoRound(object):)model = _handle_moe_model(model)# self.model_orig_dtype = model.dtype+ self.enable_alg_ext = enable_alg_extself.device = detect_device(device) ##must place after llm_load_model, because this one will convert auto## important tuning hype-parameters@@ -2566,10 +2568,35 @@ class AutoRound(object):elif isinstance(input_others[key], list):for i in range(len(input_others[key])):to_dtype(input_others[key][i], tmp_dtype)- if self.enable_torch_compile:- quant_block = compile_func(self.quantize_block, device)++ if (+ self.act_bits > 8+ and self.sym+ and self.enable_alg_ext+ and self.super_group_size is None+ and self.data_type.startswith("int")+ ):+ try:+ from auto_round.alg_ext import quantize_block_ext++ AutoRound.quantize_block_ext = quantize_block_ext+ quantize_block = self.quantize_block_ext ##must use self.quantize_block_ext+ if self.bits > 2:+ logger.warning(+ "algorithm extension has only undergone limited validation on INT2; use with caution."+ )+ else:+ logger.info("using algorithm extension for quantization.")+ except (ImportError, ModuleNotFoundError):+ quantize_block = self.quantize_block+ if self.enable_torch_compile:+ quantize_block = compile_func(quantize_block, device)+ else:+ quantize_block = quantize_blockelse:- quant_block = self.quantize_block+ quantize_block = self.quantize_block+ if self.enable_torch_compile:+ quantize_block = compile_func(quantize_block, device)if pbar is None:pbar = tqdm(range(0, len(block_names), nblocks))@@ -2590,7 +2617,7 @@ class AutoRound(object):if not self.model.device.type == "meta" or self.low_cpu_mem_usage:m = m.to(device)- q_input, input_ids = quant_block(+ q_input, input_ids = quantize_block(m,input_ids,input_others,diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 08b693c26..76177e5bb 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -198,6 +198,8 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")+ self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")+self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")@@ -565,6 +567,7 @@ def tune(args):super_bits=args.super_bits,disable_opt_rtn=args.disable_opt_rtn,disable_deterministic_algorithms=args.disable_deterministic_algorithms,+ enable_alg_ext=args.enable_alg_ext,**mllm_kwargs,)@@ -713,6 +716,8 @@ def tune(args):args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype)st = time.time()+ if "llama" in args.model.lower():+ model_args += ",add_bos_token=True"res = simple_evaluate(model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 4f47a3aa4..b9462a500 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -15,11 +15,11 @@import collections.abcimport copyimport gc+import importlibimport jsonimport loggingimport osimport re-import subprocessimport sysfrom collections import UserDictfrom enum import Enum@@ -27,7 +27,6 @@ from functools import lru_cachefrom typing import Any, Tuple, Unionimport cpuinfo-import psutilimport torchimport transformersfrom packaging import version@@ -129,10 +128,6 @@ fh = logging.StreamHandler()fh.setFormatter(AutoRoundFormatter())logger.addHandler(fh)-import importlib--import transformers-class LazyImport(object):"""Lazy import python module till use."""diff --git a/docs/alg_202508.md b/docs/alg_202508.mdnew file mode 100644index 000000000..3fb74ace8--- /dev/null+++ b/docs/alg_202508.md@@ -0,0 +1,14 @@+We use **lm-eval** for evaluation. For LLaMA, we enabled `add_bos_token` and+`removed @use_kernel_forward_from_hub("RMSNorm")`+in [modeling_llama.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L52C1-L52C40)+to stabilize accuracy during evaluation. All other settings follow the default configurations of AutoRound and lm-eval.++| Qwen3-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande |+|----------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------|+| origin | 0.4373 | 0.4019 | 0.4437 | 0.4215 | 0.4826 | 0.5474 | 0.263 | 0.3072 | 0.6314 |+| new | 0.4787 | 0.4275 | 0.4516 | 0.5944 | 0.5181 | 0.5773 | 0.2807 | 0.3305 | 0.6496 |++| Llama3.1-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande |+|-------------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------|+| origin | 0.382 | 0.3635 | 0.4562 | 0.1622 | 0.5069 | 0.4411 | 0.1661 | 0.3207 | 0.6393 |+| new | 0.4166 | 0.3712 | 0.4729 | 0.2039 | 0.5946 | 0.4981 | 0.2163 | 0.3011 | 0.6748 |\ No newline at end of file