PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of auto_round exporting (#772)
Heng Guod01704da2f90
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex cc8a526b8..4d693ece2 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1747,10 +1747,21 @@ class AutoRound(object):has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)# Iterate through all modules in the model+ is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats)for n, m in self.model.named_modules():# Skip unsupported typesif not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:- continue+ if n in self.layer_config:+ if not isinstance(m, torch.nn.Embedding):+ logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.")+ self.layer_config.pop(n)+ continue+ if not is_gguf:+ if not check_to_quantized(layer_config[n]):+ self.layer_config.pop(n)+ continue+ else:+ continue# If the layer is not in the config and is part of a quantization block, use default configurationif n not in layer_config.keys() and n in layers_in_blocks:diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 1f2b02d9b..43dbe2365 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -3,15 +3,15 @@ import shutilimport sysimport unittest-from auto_round.low_cpu_mem import get_module-sys.path.insert(0, "../..")+import torchfrom _test_helpers import model_inferfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.eval.evaluation import simple_evaluate_user_model+from auto_round.low_cpu_mem import get_moduleclass LLMDataLoader:@@ -359,6 +359,25 @@ class TestAutoRound(unittest.TestCase):model_infer(model, tokenizer)shutil.rmtree(self.save_folder)+ def test_embed_quant(self):+ bits, group_size, sym = 4, 128, True+ model_name = "facebook/opt-125m"+ layer_config = {+ "model.decoder.embed_tokens": {"bits": 4},+ }+ autoround = AutoRound(+ model_name,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ nsamples=3,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ autoround.quantize()+def test_fallback_layers(self):bits, group_size, sym = 4, 128, Truemodel_name = "facebook/opt-125m"@@ -368,6 +387,7 @@ class TestAutoRound(unittest.TestCase):layer_config = {"model.decoder.layers.0.self_attn.q_proj": {"bits": 16},"model.decoder.layers.1.self_attn.k_proj": {"bits": 16},+ "model.decoder.embed_tokens": {"bits": 16},}autoround = AutoRound(model,