Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix alg_ext moe and model str input bug (#751)

Wenhua Cheng8f84bcb79f4f
diff --git a/README.md b/README.md
index cfd900665..a23785a0a 100644
--- a/README.md
+++ b/README.md
@@ -28,10 +28,9 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
## 🆕 What's New
-[202508] AutoRound now provides experimental support for an improved INT2 algorithm via `--enable_alg_ext`. See this [documentation](./docs/alg_202508.md)
+[2025/08] AutoRound now provides experimental support for an improved INT2 algorithm via `--enable_alg_ext`. See this [documentation](./docs/alg_202508.md)
for some accuracy results.
-
[2025/07] AutoRound now offers experimental support for **GGUF** format, and recommends using optimized RTN mode (--iters 0) for
all bits other than 3 bits. Example
models: [Intel/Qwen3-235B-A22B-q2ks-mixed-ar](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-ar)
diff --git a/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so b/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so
index 2798b41d8..8cbc74494 100644
Binary files a/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so and b/auto_round/alg_ext.cpython-310-x86_64-linux-gnu.so differ
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 96063ca6a..15ee357b5 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -228,17 +228,12 @@ class AutoRound(object):
self.data_type = data_type
tmp_bits = infer_bits_by_data_type(self.data_type)
if tmp_bits < 16 and tmp_bits != bits:
- logger.warning(
- f"Bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}."
- )
+ logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
self.bits = tmp_bits
self.group_size = group_size
self.sym = sym
self.layer_config = {} if layer_config is None else layer_config
self.to_quant_block_names = to_quant_block_names
- if not hasattr(self, "quant_block_list"):
- all_blocks = get_block_names(model)
- self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
# Activation
self.act_group_size = act_group_size if act_group_size is not None else group_size
@@ -256,7 +251,7 @@ class AutoRound(object):
if tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
self.act_bits = tmp_act_bits
logger.warning(
- f" 'act_data_type' do not"
+ f"`act_data_type` do not"
f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
)
@@ -316,6 +311,9 @@ class AutoRound(object):
self.model = model.eval()
self.tokenizer = tokenizer
self.shared_cache_keys = get_shared_keys(self.model)
+ if not hasattr(self, "quant_block_list"):
+ all_blocks = get_block_names(model)
+ self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
# Set device, must place after model loading
self.device = detect_device(device) # must place after llm_load_model, because this one will convert auto
@@ -1290,7 +1288,7 @@ class AutoRound(object):
clear_memory()
cnt = 1
cnt += 1
- ##convert remainning fp8
+ # Convert remaining fp8
if hasattr(self.model, "is_fp8"):
convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
self.quantized = True
@@ -1482,10 +1480,13 @@ class AutoRound(object):
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
clear_memory()
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changed
+ accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
logger.info("caching done")
- pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
+ if len(all_blocks) > 1:
+ pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
+ else:
+ pbar = None # move the alg warning outside pbar
for block_names in all_blocks:
inputs = all_inputs[block_names[0]]
@@ -1527,7 +1528,6 @@ class AutoRound(object):
f"Expected exactly one packing format when 'is_packing_immediate' is True, "
f"but got {len(self.formats)} formats."
)
- pbar.close()
self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately
@@ -1686,14 +1686,13 @@ class AutoRound(object):
layer_config[match_name] = val
else:
tmp_m = get_module(self.model, name)
- if not isinstance(tmp_m, torch.nn.Embedding): ##TODO not good code style
+ if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style
raise ValueError(f"key {name} in layer_config is invalid, please have a double check")
has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
# Iterate through all modules in the model
for n, m in self.model.named_modules():
-
# Skip unsupported types
if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
continue
@@ -1703,12 +1702,28 @@ class AutoRound(object):
layer_config[n] = {}
for key in keys:
layer_config[n][key] = getattr(self, key)
+
# If the layer is partially configured, fill in missing values
elif n in layer_config.keys():
for key in keys:
if key not in layer_config[n].keys():
layer_config[n][key] = getattr(self, key)
layer_config[n]["fixed_by_user"] = True
+ # Check dtype and act_dtype
+ tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])
+ if tmp_bits != layer_config[n]["bits"]:
+ logger.warning(
+ f"'data_type' do not match the specified 'bits' setting for {n}."
+ f" Resetting 'bits' to {tmp_bits}."
+ )
+ layer_config[n]["bits"] = tmp_bits
+ tmp_act_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])
+ if tmp_act_bits != layer_config[n]["act_bits"]:
+ logger.warning(
+ f"'act_data_type' do not match the specified 'act_bits' setting for {n}."
+ f" Resetting 'act_bits' to {tmp_act_bits}."
+ )
+ layer_config[n]["act_bits"] = tmp_act_bits
# If the layer is not in the config and not part of a quantization block,
# use default configuration and set specific values
else:
@@ -2676,7 +2691,7 @@ class AutoRound(object):
PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])
pbar.set_description("Quantizing done")
pbar.update(1)
-
+ pbar.close()
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
for n, m in self.model.named_modules():
if hasattr(m, "name"):
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 9dcfbafb3..4d55869cd 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -3,6 +3,8 @@ import shutil
import sys
import unittest
+from auto_round.low_cpu_mem import get_module
+
sys.path.insert(0, "../..")
import torch
from _test_helpers import model_infer
@@ -35,6 +37,14 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
+ def test_bits_setting(self):
+ layer_config = {"model.decoder.layers.0.self_attn.k_proj": {"data_type": "mx_fp8", "group_size": 32}}
+ autoround = AutoRound("facebook/opt-125m", iters=2, seqlen=2, nsamples=1, layer_config=layer_config)
+ autoround.quantize()
+ module = get_module(autoround.model, "model.decoder.layers.0.self_attn.k_proj")
+ if module.bits != 8:
+ raise ValueError(f"Expected bits to be 8, but got {module.bits}")
+
def test_remove_whole_block(self):
layer_config = {
"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index d05273b69..d3f255397 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -71,25 +71,25 @@ class TestGGUF(unittest.TestCase):
# self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23)
shutil.rmtree("./saved", ignore_errors=True)
- def test_q4_1(self):
- bits, group_size, sym = 4, 32, False
- autoround = AutoRound(
- self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1
- )
- quantized_model_path = "./saved"
-
- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")
- gguf_file = os.listdir(quantized_model_path)[0]
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
- text = "There is a girl who likes adventure,"
- inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
- print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
-
- # from auto_round.eval.evaluation import simple_evaluate_user_model
- # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")
- # # 0.23
- # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22)
- shutil.rmtree("./saved", ignore_errors=True)
+ # def test_q4_1(self):
+ # bits, group_size, sym = 4, 32, False
+ # autoround = AutoRound(
+ # self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1
+ # )
+ # quantized_model_path = "./saved"
+ #
+ # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")
+ # gguf_file = os.listdir(quantized_model_path)[0]
+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
+ # text = "There is a girl who likes adventure,"
+ # inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ #
+ # # from auto_round.eval.evaluation import simple_evaluate_user_model
+ # # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")
+ # # # 0.23
+ # # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22)
+ # shutil.rmtree("./saved", ignore_errors=True)
def test_func(self):
bits, group_size, sym = 4, 128, True
@@ -114,20 +114,20 @@ class TestGGUF(unittest.TestCase):
print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
shutil.rmtree("./saved", ignore_errors=True)
- model_name = "Qwen/Qwen2.5-1.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- autoround = AutoRound(
- model,
- self.tokenizer,
- bits=3,
- group_size=16,
- sym=True,
- iters=1,
- nsamples=1,
- data_type="int_sym_dq",
- super_group_size=16,
- super_bits=6,
- )
+ # model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+ # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ # autoround = AutoRound(
+ # model,
+ # self.tokenizer,
+ # bits=3,
+ # group_size=16,
+ # sym=True,
+ # iters=1,
+ # nsamples=1,
+ # data_type="int_sym_dq",
+ # super_group_size=16,
+ # super_bits=6,
+ # )
quantized_model_path = "./saved"
# autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")
# from auto_round.eval.evaluation import simple_evaluate_user_model
@@ -137,53 +137,54 @@ class TestGGUF(unittest.TestCase):
# self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5)
shutil.rmtree("./saved", ignore_errors=True)
- def test_q5_k(self):
- model_name = "Qwen/Qwen2.5-1.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- autoround = AutoRound(
- model,
- self.tokenizer,
- bits=5,
- group_size=32,
- sym=False,
- iters=1,
- nsamples=1,
- data_type="int_asym_dq",
- super_group_size=8,
- super_bits=6,
- )
- quantized_model_path = "./saved"
- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")
- gguf_file = os.listdir("saved")[0]
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
- text = "There is a girl who likes adventure,"
- inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
- print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
+ #
+ # def test_q5_k(self):
+ # model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+ # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ # autoround = AutoRound(
+ # model,
+ # self.tokenizer,
+ # bits=5,
+ # group_size=32,
+ # sym=False,
+ # iters=1,
+ # nsamples=1,
+ # data_type="int_asym_dq",
+ # super_group_size=8,
+ # super_bits=6,
+ # )
+ # quantized_model_path = "./saved"
+ # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")
+ # gguf_file = os.listdir("saved")[0]
+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
+ # text = "There is a girl who likes adventure,"
+ # inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ # shutil.rmtree("./saved", ignore_errors=True)
- def test_q6_k(self):
- model_name = "Qwen/Qwen2.5-1.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- autoround = AutoRound(
- model,
- self.tokenizer,
- bits=6,
- group_size=16,
- sym=True,
- iters=1,
- nsamples=1,
- data_type="int_sym_dq",
- super_group_size=16,
- super_bits=8,
- )
- quantized_model_path = "./saved"
- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k")
- gguf_file = os.listdir("saved")[0]
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
- text = "There is a girl who likes adventure,"
- inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
- print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
+ # def test_q6_k(self):
+ # model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+ # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ # autoround = AutoRound(
+ # model,
+ # self.tokenizer,
+ # bits=6,
+ # group_size=16,
+ # sym=True,
+ # iters=1,
+ # nsamples=1,
+ # data_type="int_sym_dq",
+ # super_group_size=16,
+ # super_bits=8,
+ # )
+ # quantized_model_path = "./saved"
+ # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k")
+ # gguf_file = os.listdir("saved")[0]
+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
+ # text = "There is a girl who likes adventure,"
+ # inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ # shutil.rmtree("./saved", ignore_errors=True)
def test_gguf_baseline(self):
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -208,28 +209,28 @@ class TestGGUF(unittest.TestCase):
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
shutil.rmtree("./saved", ignore_errors=True)
-
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- autoround = AutoRound(
- model,
- self.tokenizer,
- bits=5,
- group_size=32,
- sym=True,
- iters=0,
- nsamples=8,
- data_type="int_asym_dq",
- super_group_size=8,
- super_bits=6,
- disable_opt_rtn=True,
- )
- quantized_model_path = "./saved"
- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto")
- text = "There is a girl who likes adventure,"
- inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
- print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
+ #
+ # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ # autoround = AutoRound(
+ # model,
+ # self.tokenizer,
+ # bits=5,
+ # group_size=32,
+ # sym=True,
+ # iters=0,
+ # nsamples=8,
+ # data_type="int_asym_dq",
+ # super_group_size=8,
+ # super_bits=6,
+ # disable_opt_rtn=True,
+ # )
+ # quantized_model_path = "./saved"
+ # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake")
+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto")
+ # text = "There is a girl who likes adventure,"
+ # inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ # shutil.rmtree("./saved", ignore_errors=True)
def test_q4_k_m(self):
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -283,7 +284,8 @@ class TestGGUF(unittest.TestCase):
def test_all_format(self):
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
python_path = sys.executable
- for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]:
+ # for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]:
+ for gguf_format in ["gguf:q4_k_m"]:
res = os.system(
f"cd ../.. && {python_path} -m auto_round --model {model_name} "
f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}"