Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bug of auto_round exporting (#772)

Heng Guod01704da2f90
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index cc8a526b8..4d693ece2 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1747,10 +1747,21 @@ class AutoRound(object):
has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
# Iterate through all modules in the model
+ is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats)
for n, m in self.model.named_modules():
# Skip unsupported types
if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
- continue
+ if n in self.layer_config:
+ if not isinstance(m, torch.nn.Embedding):
+ logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.")
+ self.layer_config.pop(n)
+ continue
+ if not is_gguf:
+ if not check_to_quantized(layer_config[n]):
+ self.layer_config.pop(n)
+ continue
+ else:
+ continue
# If the layer is not in the config and is part of a quantization block, use default configuration
if n not in layer_config.keys() and n in layers_in_blocks:
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 1f2b02d9b..43dbe2365 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -3,15 +3,15 @@ import shutil
import sys
import unittest
-from auto_round.low_cpu_mem import get_module
-
sys.path.insert(0, "../..")
+
import torch
from _test_helpers import model_infer
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
from auto_round.eval.evaluation import simple_evaluate_user_model
+from auto_round.low_cpu_mem import get_module
class LLMDataLoader:
@@ -359,6 +359,25 @@ class TestAutoRound(unittest.TestCase):
model_infer(model, tokenizer)
shutil.rmtree(self.save_folder)
+ def test_embed_quant(self):
+ bits, group_size, sym = 4, 128, True
+ model_name = "facebook/opt-125m"
+ layer_config = {
+ "model.decoder.embed_tokens": {"bits": 4},
+ }
+ autoround = AutoRound(
+ model_name,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ nsamples=3,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ autoround.quantize()
+
def test_fallback_layers(self):
bits, group_size, sym = 4, 128, True
model_name = "facebook/opt-125m"
@@ -368,6 +387,7 @@ class TestAutoRound(unittest.TestCase):
layer_config = {
"model.decoder.layers.0.self_attn.q_proj": {"bits": 16},
"model.decoder.layers.1.self_attn.k_proj": {"bits": 16},
+ "model.decoder.embed_tokens": {"bits": 16},
}
autoround = AutoRound(
model,