PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix reload issue for lm_head quantization and update CI to cover it (#1563)
Xin He13fbffee25f3
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex e15f918aa..0a2352e2a 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1859,15 +1859,14 @@ class BaseCompressor(object):)pbar.set_description("Quantizing done")pbar.close()+ if self.low_cpu_mem_usage:+ self._offloader.reload(self.model)self._quantize_layers(layer_names, all_inputs)convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device, to_cpu=True)if self.is_immediate_saving:shard_writer(self, is_finalize=True)- if self.low_cpu_mem_usage:- self._offloader.reload(self.model)-end_time = time.time()cost_time = end_time - start_timelogger.info(f"quantization tuning time {cost_time}")diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex 0ed63dda4..701e02ea1 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -253,19 +253,16 @@ class TestAutoRound:)autoround.quantize()- def test_lm_head_layer_config_way(self, dataloader):+ def test_lm_head_layer_config_way(self, tiny_untied_qwen_model_path, dataloader):bits, group_size, sym = 4, -1, Falselayer_config = {"lm_head": {"data_type": "int"}}autoround = AutoRound(- self.model,- self.tokenizer,+ tiny_untied_qwen_model_path,bits=bits,group_size=group_size,sym=sym,- iters=2,- seqlen=10,- enable_minmax_tuning=False,- enable_quanted_input=False,+ iters=1,+ seqlen=1,dataset=dataloader,layer_config=layer_config,)@@ -637,21 +634,13 @@ class TestAutoRound:def test_quant_lm_head(self, tiny_untied_qwen_model_path):model_name = tiny_untied_qwen_model_path- ar = AutoRound(model_name, quant_lm_head=True, iters=0, seqlen=8, nsamples=1, disable_opt_rtn=True)- ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(self.save_folder, device_map="cpu")- assert "lm_head" in model.config.quantization_config.extra_config- assert model.config.quantization_config.extra_config["lm_head"]["bits"] == 4-- layer_config = {"lm_head": {"bits": 4}}ar = AutoRound(model_name,- quant_lm_head=False,+ quant_lm_head=True,iters=0,seqlen=8,nsamples=1,disable_opt_rtn=True,- layer_config=layer_config,)ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")model = AutoModelForCausalLM.from_pretrained(self.save_folder, device_map="cpu")